path: root/screener.js

const { parse } = require('node:path');
const { default: test } = require('node:test');

fs = require('node:fs');

// this gets the HTML page of the SP500 list from slickcharts.com
const getSPHTML = async () => {

  const response = await fetch('https://www.slickcharts.com/sp500');
  if (!response.ok) {
    throw new Error('Network response was not ok');
  }
    const text = await response.text();
    return text;
}

// this parses the HTML of the SP500 list to tickers
const parseHTMLToTickers = (html) => {
    // get the tickers by slicing on all `/symbol/` occurrences 
    // (with the special format before it to only get one occurance)
    const tickers = [];
    html.split('nowrap;"><a href="/symbol/').slice(1).forEach(item => {
        // get the ticker from the item (before the next ")"
        const ticker = item.split('"')[0];
        const name = item.split('">')[1].split('</a>')[0];

        // get weight of item using the %
        let weight = item.search(/<td>([\d.]+)%<\/td>/);
        if (weight === -1) {
            console.warn(`Ticker ${ticker} does not have a valid weight, skipping.`);
            return;
        }
        weight = parseFloat(item.slice(weight + 4, item.indexOf('</td>', weight)));

        if (ticker && name && weight) {
            tickers.push({ name, symbol: ticker, weight });
        }
    });

    // update the tickers file with the new tickers
    if (fs.existsSync('sp500_tickers.json')) {
        fs.unlinkSync('sp500_tickers.json');
    }
    fs.writeFileSync('sp500_tickers.json', JSON.stringify(tickers, null, 2));
    console.log(`Saved ${tickers.length} tickers to sp500_tickers.json`);
    return tickers;
}

const getTickersFromFile = () => {
    if (!fs.existsSync('sp500_tickers.json')) {
        console.error('sp500_tickers.json file does not exist. Please run the script to fetch tickers first.');
        return [];
    }
    const data = fs.readFileSync('sp500_tickers.json', 'utf8');
    return JSON.parse(data);
}

// get the JSON history data from the symbol
const getSymbolHistory = async (symbol) => {
    // clean the symbol (reaplce . with dash)
    symbol = symbol.replace(/\./g, '-');
    const parameters = 'interval=1d&includePrePost=true&events=div%7Csplit%7Cearn&lang=en-US&region=US&range=6mo';
    const response = await fetch(`https://query1.finance.yahoo.com/v8/finance/chart/${symbol}?${parameters}`);
    if (!response.ok) {
        console.error(`Network response was not ok for symbol: ${symbol}. Status: ${response.status}`);
        return {};
    }
    const data = await response.json();

    return data;
}

const getSectorMap = () => {
    // pull from the sector map tsv file
    if (!fs.existsSync('sector-map.tsv')) {
        console.error('sector-map.tsv file does not exist. Please run the script to fetch sectors first.');
        return 'Unknown';
    }
    const sectorMap = fs.readFileSync('sector-map.tsv', 'utf8');
    const lines = sectorMap.split('\n');
    const sectorMapObj = {};
    lines.forEach((line, index) => {
        if (index === 0) return; // skip the header line
        // split the line by comma and get the name, ticker, sector, and subSector
        const [name, ticker, sector, subSector] = line.split('\t');
        sectorMapObj[ticker.trim()] = [sector.trim(), subSector.trim()];
    });
    return sectorMapObj;
}

const getHistoriesForEachTicker = async (tickers) => {
    // use Promise.all to fetch all histories concurrently
    const histories = await Promise.all(tickers.map(ticker => getSymbolHistory(ticker.symbol)));

    // zip the histories with the tickers
    const zippedHistories = histories.map((history, index) => ({
        ...tickers[index],
        history
    }));
    return zippedHistories;
}

// test getting the histories for all the symbols
const testGetHistories = async () => {
    try {
        // const html = await getSPHTML();
        // const tickers = parseHTMLToTickers(html);
        // console.log('SP500 Tickers:', tickers);

        // get tickers from file
        const tickers = getTickersFromFile();
        if (tickers.length === 0) {
            console.error('No tickers found. Please ensure sp500_tickers.json exists and is populated.');
            return;
        }
        console.log(`Found ${tickers.length} tickers in sp500_tickers.json`);

        // get histories for each symbol
        const histories = await getHistoriesForEachTicker(tickers);
        console.log('Histories fetched for all symbols:', histories.length);

        // save histories to a file
        if (fs.existsSync('sp500_histories.json')) {
            fs.unlinkSync('sp500_histories.json');
        }
        fs.writeFileSync('sp500_histories.json', JSON.stringify(histories, null, 2));
        console.log(`Saved ${histories.length} histories to sp500_histories.json`);
        
        // print first 5 and last 5 histories
        console.log('First 5 histories:');
        histories.slice(0, 5).forEach(h => console.log(h.symbol, h.history));
        // console.log('Last 5 histories:', histories.slice(-5));
    } catch (error) {
        console.error('Error fetching SP500 histories:', error);
    }
};

const formatDataFromHistories = (histories) => {
    // format the data from the histories to a more readable format
    const csv_headers = ['Ticker', 'Name', '% Weight', 'Sector', 'SubSector', 'RSI (14)', 'MACD (Histogram Value)', '1W', '1M', '3M', '6M'];
    const csv_final = [csv_headers];

    // get the sector map
    const sectorMap = getSectorMap();
    
    histories.forEach(history => {

        // Tickern, name weight, sector from html pull
        const { symbol, name, weight } = history;
        const sector = sectorMap[symbol] ? sectorMap[symbol][0] : 'Unknown';

        // Get RSI, MACD from helper
        const timestamps = history.history.chart.result[0].timestamp;
        const prices = history.history.chart.result[0].indicators.quote[0].close;
        const rsi = calculateRSI(prices);
        const macd = calculateMACD(prices);

        // print first 5 timestamps and prices for debugging
        console.log('First 5 timestamps:', timestamps.slice(0, 5).map(ts => new Date(ts * 1000).toLocaleDateString()));
        console.log('First 5 prices:', prices.slice(0, 5));
        
        const currentPrice = prices[prices.length - 1];
        // Directly calculate the percentage changes for 1W, 1M, 3M, and 6M\
        const oneWeekAgoPrice = prices[prices.length - 6]; // 7 days ago
        const oneWeekChange = ((currentPrice - oneWeekAgoPrice) / oneWeekAgoPrice) * 100;

        const oneMonthAgoPrice = prices[prices.length - 21]; // 21 days ago (trading days in a month)
        const oneMonthChange = ((currentPrice - oneMonthAgoPrice) / oneMonthAgoPrice) * 100;

        const threeMonthsAgoPrice = prices[prices.length - 63]; // 63 days ago (trading days in 3 months)
        const threeMonthChange = ((currentPrice - threeMonthsAgoPrice) / threeMonthsAgoPrice) * 100;

        const sixMonthsAgoPrice = prices[0]; // last 6 months is the first price in the array
        const sixMonthChange = ((currentPrice - sixMonthsAgoPrice) / sixMonthsAgoPrice) * 100;
        
        const mappedValues = {
            Ticker: symbol,
            Name: name.replace(',', ' '), // replace commas in name with spaces for CSV
            '% Weight': weight,
            'Sector': sectorMap[symbol] ? sectorMap[symbol][0] : 'Unknown',
            'Subsector': sectorMap[symbol] ? sectorMap[symbol][1] : 'Unknown',
            'RSI (14)': rsi,
            'MACD (Histogram Value)': macd,
            '1W': oneWeekChange.toFixed(3) + '%',
            '1M': oneMonthChange.toFixed(3) + '%',
            '3M': threeMonthChange.toFixed(3) + '%',
            '6M': sixMonthChange.toFixed(3) + '%'
        };

        // pushed the mapped values to the formatted data
        csv_final.push(Object.values(mappedValues));
    });

    // write the formatted data to a CSV file
    const csvContent = csv_final.map(e => e.join(',')).join('\n');
    if (fs.existsSync('sp500_formatted_data.csv')) {
        fs.unlinkSync('sp500_formatted_data.csv');
    }
    fs.writeFileSync('sp500_formatted_data.csv', csvContent);
    console.log('Formatted data saved to sp500_formatted_data.csv');
    return csv_final;
};
// testGetHistories();

const main = async () => {
    try {
        // gt the test histories from the file
        // const histories = fs.readFileSync('sp500_histories.json', 'utf8');
        // const parsedHistories = JSON.parse(histories);
        // console.log(`Loaded ${parsedHistories.length} histories from sp500_histories.json`);

        // get tickers from file
        const tickers = getTickersFromFile();
        if (tickers.length === 0) {
            console.error('No tickers found. Please ensure sp500_tickers.json exists and is populated.');
            return;
        }
        console.log(`Found ${tickers.length} tickers in sp500_tickers.json`);
        // get histories for each symbol
        const parsedHistories = await getHistoriesForEachTicker(tickers);
        console.log(`Fetched histories for ${parsedHistories.length} symbols.`);


        // format the data from the histories
        const formattedData = formatDataFromHistories(parsedHistories);
        console.log('Formatted data:', formattedData.slice(0, 5)); // Print first 5 entries for brevity

    } catch (error) {
        console.error('Error in main function:', error);
    }
}

const calculateMACD = (prices, shortPeriod = 12, longPeriod = 26, signalPeriod = 9) => {
    // Helper function to calculate the Exponential Moving Average (EMA)
    const exponentialMovingAverage = (data, period) => {
        const k = 2 / (period + 1);
        let ema = [data[0]]; // Start with the first price as the initial EMA

        for (let i = 1; i < data.length; i++) {
            const currentEma = (data[i] * k) + (ema[i - 1] * (1 - k));
            ema.push(currentEma);
        }
        return ema;
    }

    // Calculate the short and long periods
    const ema12 = exponentialMovingAverage(prices, shortPeriod);
    const ema26 = exponentialMovingAverage(prices, longPeriod);

    // Calcualte the MACD line
    const macdLine = ema12.map((value, index) => value - ema26[index])

    // Calculate the signal line
    const signalLine = exponentialMovingAverage(macdLine, signalPeriod);

    // Calculate the MACD histogram
    const macdHistogram = macdLine.map((value, index) => value - signalLine[index]);

    // Return the last value of the MACD histogram
    return macdHistogram[macdHistogram.length - 1];
}


const calculateRSI = (prices, period = 14) => {
    // calculate the first RSI within our period
    let gains = [];
    let losses = [];

    for (let i = 0; i < period; i++) {
        const change = prices[i + 1] - prices[i];
        if (change > 0) {
            gains.push(change);
            losses.push(0);
        } else {
            losses.push(Math.abs(change));
            gains.push(0);
        }
    }

    const averageGain = gains.reduce((a, b) => a + b, 0) / period;
    const averageLoss = losses.reduce((a, b) => a + b, 0) / period;
    if (averageLoss === 0) {
        console.log('No losses in the period, RSI is 100');
        return 100; // RSI is 100 if there are no losses
    }
    const firstRSI = 100 - (100 / (1 + (averageGain / averageLoss)));
    if (isNaN(firstRSI)) {
        console.error('Calculated RSI is NaN, returning 0');
        return 0; // Return 0 if RSI calculation fails
    }

    const RSIs = [firstRSI];
    console.log(`Initial RSI for the first ${period} data points: ${firstRSI}`);

    // Calculate the RSI for the rest of the data points
    let previousAverageGain = averageGain;
    let previousAverageLoss = averageLoss;

    for (let i = period; i < prices.length - 1; i++) {
        const change = prices[i + 1] - prices[i];
        let gain = 0;
        let loss = 0;

        if (change > 0) {
            gain = change;
        } else {
            loss = Math.abs(change);
        }

        // Calculate the new average gain and loss
        previousAverageGain = (previousAverageGain * (period - 1) + gain) / period;
        previousAverageLoss = (previousAverageLoss * (period - 1) + loss) / period;
        if (previousAverageLoss === 0) {
            console.log('No losses in the period, RSI is 100');
            return 100; // RSI is 100 if there are no losses
        }

        // add this RSI to the list
        const rsi = 100 - (100 / (1 + (previousAverageGain / previousAverageLoss)));
        RSIs.push(rsi);
    }

    // Return the last calculated RSI
    return RSIs[RSIs.length - 1];
}

// test rsi calculation on one stock
const testRSI = async () => {
    const symbol = 'NVDA';
    const history = await getSymbolHistory(symbol);
    if (!history.chart || !history.chart.result || history.chart.result.length === 0) {
        console.error(`No data found for symbol: ${symbol}`);
        return;
    }
    const prices = history.chart.result[0].indicators.quote[0].close;
    const timestamps = history.chart.result[0].timestamp;
    // print the last 10 prices and dates
    const rsi = calculateRSI(prices);
    console.log(`RSI for ${symbol}:`, rsi);
}

const testGetSector = async () => {
    // pull for each ticker in the sp500_tickers.json file
    const tickers = getTickersFromFile();
    if (tickers.length === 0) {
        console.error('No tickers found. Please ensure sp500_tickers.json exists and is populated.');
        return;
    }
    // get the sector map
    const sectorMap = getSectorMap();
    tickers.forEach(async (ticker) => {
        const sector = sectorMap[ticker.symbol] ? sectorMap[ticker.symbol][0] : 'Unknown';
        console.log(`Ticker: ${ticker.symbol}, Sector: ${sector}`);
    });
}

// testGetSector();

main();