fix fetch import, update server.js, git ignore gen file, remove screener.js and upload.js

1 files changed, 0 insertions, 353 deletions

diff --git a/screener.js b/screener.js
deleted file mode 100644
index 85e6183..0000000
--- a/screener.js
+++ /dev/null
@@ -1,353 +0,0 @@
-import fs from 'fs';
-
-// this gets the HTML page of the SP500 list from slickcharts.com
-const getSPHTML = async () => {
-  const response = await fetch('https://www.slickcharts.com/sp500');
-  if (!response.ok) {
-    throw new Error('Network response was not ok');
-  }
-    const text = await response.text();
-    return text;
-}
-
-// this parses the HTML of the SP500 list to tickers
-const parseHTMLToTickers = (html) => {
-    // get the tickers by slicing on all `/symbol/` occurrences 
-    // (with the special format before it to only get one occurance)
-    const tickers = [];
-    html.split('nowrap;"><a href="/symbol/').slice(1).forEach(item => {
-        // get the ticker from the item (before the next ")"
-        const ticker = item.split('"')[0];
-        const name = item.split('">')[1].split('</a>')[0];
-
-        // get weight of item using the %
-        let weight = item.search(/<td>([\d.]+)%<\/td>/);
-        if (weight === -1) {
-            console.warn(`Ticker ${ticker} does not have a valid weight, skipping.`);
-            return;
-        }
-        weight = parseFloat(item.slice(weight + 4, item.indexOf('</td>', weight)));
-
-        if (ticker && name && weight) {
-            tickers.push({ name, symbol: ticker, weight });
-        }
-    });
-
-    // update the tickers file with the new tickers
-    if (fs.existsSync('sp500_tickers.json')) {
-        fs.unlinkSync('sp500_tickers.json');
-    }
-    fs.writeFileSync('sp500_tickers.json', JSON.stringify(tickers, null, 2));
-    // console.log(`Saved ${tickers.length} tickers to sp500_tickers.json`);
-    return tickers;
-}
-
-const getTickersFromFile = () => {
-    if (!fs.existsSync('sp500_tickers.json')) {
-        console.error('sp500_tickers.json file does not exist. Please run the script to fetch tickers first.');
-        return [];
-    }
-    const data = fs.readFileSync('sp500_tickers.json', 'utf8');
-    return JSON.parse(data);
-}
-
-// get the JSON history data from the symbol
-const getSymbolHistory = async (symbol) => {
-    // clean the symbol (reaplce . with dash)
-    symbol = symbol.replace(/\./g, '-');
-    const parameters = 'interval=1d&includePrePost=true&events=div%7Csplit%7Cearn&lang=en-US&region=US&range=6mo';
-    const response = await fetch(`https://query1.finance.yahoo.com/v8/finance/chart/${symbol}?${parameters}`);
-    if (!response.ok) {
-        console.error(`Network response was not ok for symbol: ${symbol}. Status: ${response.status}`);
-        return {};
-    }
-    const data = await response.json();
-
-    return data;
-}
-
-const getSectorMap = () => {
-    // pull from the sector map tsv file
-    if (!fs.existsSync('sector-map.tsv')) {
-        console.error('sector-map.tsv file does not exist. Please run the script to fetch sectors first.');
-        return 'Unknown';
-    }
-    const sectorMap = fs.readFileSync('sector-map.tsv', 'utf8');
-    const lines = sectorMap.split('\n');
-    const sectorMapObj = {};
-    lines.forEach((line, index) => {
-        if (index === 0) return; // skip the header line
-        // split the line by comma and get the name, ticker, sector, and subSector
-        const [name, ticker, sector, subSector] = line.split('\t');
-        sectorMapObj[ticker.trim()] = [sector.trim(), subSector.trim(), name.trim()];
-    });
-    return sectorMapObj;
-}
-
-const getHistoriesForEachTicker = async (tickers) => {
-    // use Promise.all to fetch all histories concurrently
-    const histories = await Promise.all(tickers.map(ticker => getSymbolHistory(ticker.symbol)));
-
-    // zip the histories with the tickers
-    const zippedHistories = histories.map((history, index) => ({
-        ...tickers[index],
-        history
-    }));
-    return zippedHistories;
-}
-
-// test getting the histories for all the symbols
-const testGetHistories = async () => {
-    try {
-        // const html = await getSPHTML();
-        // const tickers = parseHTMLToTickers(html);
-        // console.log('SP500 Tickers:', tickers);
-
-        // get tickers from file
-        const tickers = getTickersFromFile();
-        if (tickers.length === 0) {
-            console.error('No tickers found. Please ensure sp500_tickers.json exists and is populated.');
-            return;
-        }
-        console.log(`Found ${tickers.length} tickers in sp500_tickers.json`);
-
-        // get histories for each symbol
-        const histories = await getHistoriesForEachTicker(tickers);
-        console.log('Histories fetched for all symbols:', histories.length);
-
-        // save histories to a file
-        if (fs.existsSync('sp500_histories.json')) {
-            fs.unlinkSync('sp500_histories.json');
-        }
-        fs.writeFileSync('sp500_histories.json', JSON.stringify(histories, null, 2));
-        console.log(`Saved ${histories.length} histories to sp500_histories.json`);
-        
-        // print first 5 and last 5 histories
-        console.log('First 5 histories:');
-        histories.slice(0, 5).forEach(h => console.log(h.symbol, h.history));
-        // console.log('Last 5 histories:', histories.slice(-5));
-    } catch (error) {
-        console.error('Error fetching SP500 histories:', error);
-    }
-};
-
-const formatDataFromHistories = (histories) => {
-    // format the data from the histories to a more readable format
-    const csv_headers = ['Ticker', 'Name', '% Weight', 'Sector', 'SubSector', 'RSI (14)', 'MACD (Histogram Value)', '1W', '1M', '3M', '6M'];
-    const csv_final = [csv_headers];
-
-    // get the sector map
-    const sectorMap = getSectorMap();
-    
-    histories.forEach(history => {
-
-        // Tickern, name weight, sector from html pull
-        const { symbol, webName, weight } = history;
-        const sector = sectorMap[symbol] ? sectorMap[symbol][0] : 'Unknown';
-        const subSector = sectorMap[symbol] ? sectorMap[symbol][1] : 'Unknown';
-        const name = sectorMap[symbol] ? sectorMap[symbol][2] : webName || 'Unknown';
-
-        // Get RSI, MACD from helper
-        const timestamps = history.history.chart.result[0].timestamp;
-        const prices = history.history.chart.result[0].indicators.quote[0].close;
-        const rsi = calculateRSI(prices);
-        const macd = calculateMACD(prices);
-
-        // print first 5 timestamps and prices for debugging
-        // console.log('First 5 timestamps:', timestamps.slice(0, 5).map(ts => new Date(ts * 1000).toLocaleDateString()));
-        // console.log('First 5 prices:', prices.slice(0, 5));
-        
-        const currentPrice = prices[prices.length - 1];
-        // Directly calculate the percentage changes for 1W, 1M, 3M, and 6M\
-        const oneWeekAgoPrice = prices[prices.length - 6]; // 5 days of trading
-        const oneWeekChange = ((currentPrice - oneWeekAgoPrice) / oneWeekAgoPrice) * 100;
-
-        const oneMonthAgoPrice = prices[prices.length - 21]; // 20 days of trading (4 weeks)
-        const oneMonthChange = ((currentPrice - oneMonthAgoPrice) / oneMonthAgoPrice) * 100;
-
-        const threeMonthsAgoPrice = prices[parseInt(prices.length / 2) - 1]; // 3 months is half the length of the prices array
-        const threeMonthChange = ((currentPrice - threeMonthsAgoPrice) / threeMonthsAgoPrice) * 100;
-
-        const sixMonthsAgoPrice = prices[0]; // last 6 months is the first price in the array
-        const sixMonthChange = ((currentPrice - sixMonthsAgoPrice) / sixMonthsAgoPrice) * 100;
-        
-        const mappedValues = {
-            Ticker: symbol,
-            Name: name,
-            '% Weight': weight,
-            'Sector': sector,
-            'Subsector': subSector,
-            'RSI (14)': rsi.toFixed(3),
-            'MACD (Histogram Value)': macd.toFixed(3),
-            '1W': oneWeekChange.toFixed(3) + '%',
-            '1M': oneMonthChange.toFixed(3) + '%',
-            '3M': threeMonthChange.toFixed(3) + '%',
-            '6M': sixMonthChange.toFixed(3) + '%'
-        };
-
-        // pushed the mapped values to the formatted data
-        csv_final.push(Object.values(mappedValues));
-    });
-
-    // write the formatted data to a CSV file
-    const csvContent = csv_final.map(e => e.join('\t')).join('\n');
-    if (fs.existsSync('sp500_formatted_data.tsv')) {
-        fs.unlinkSync('sp500_formatted_data.tsv');
-    }
-    fs.writeFileSync('sp500_formatted_data.tsv', csvContent);
-    // console.log('Formatted data saved to sp500_formatted_data.tsv');
-    return csv_final;
-};
-// testGetHistories();
-
-const calculateMACD = (prices, shortPeriod = 12, longPeriod = 26, signalPeriod = 9) => {
-    // Helper function to calculate the Exponential Moving Average (EMA)
-    const exponentialMovingAverage = (data, period) => {
-        const k = 2 / (period + 1);
-        let ema = [data[0]]; // Start with the first price as the initial EMA
-
-        for (let i = 1; i < data.length; i++) {
-            const currentEma = (data[i] * k) + (ema[i - 1] * (1 - k));
-            ema.push(currentEma);
-        }
-        return ema;
-    }
-
-    // Calculate the short and long periods
-    const ema12 = exponentialMovingAverage(prices, shortPeriod);
-    const ema26 = exponentialMovingAverage(prices, longPeriod);
-
-    // Calcualte the MACD line
-    const macdLine = ema12.map((value, index) => value - ema26[index])
-
-    // Calculate the signal line
-    const signalLine = exponentialMovingAverage(macdLine, signalPeriod);
-
-    // Calculate the MACD histogram
-    const macdHistogram = macdLine.map((value, index) => value - signalLine[index]);
-
-    // Return the last value of the MACD histogram
-    return macdHistogram[macdHistogram.length - 1];
-}
-
-
-const calculateRSI = (prices, period = 14) => {
-    // calculate the first RSI within our period
-    let gains = [];
-    let losses = [];
-
-    for (let i = 0; i < period; i++) {
-        const change = prices[i + 1] - prices[i];
-        if (change > 0) {
-            gains.push(change);
-            losses.push(0);
-        } else {
-            losses.push(Math.abs(change));
-            gains.push(0);
-        }
-    }
-
-    const averageGain = gains.reduce((a, b) => a + b, 0) / period;
-    const averageLoss = losses.reduce((a, b) => a + b, 0) / period;
-    if (averageLoss === 0) {
-        console.log('No losses in the period, RSI is 100');
-        return 100; // RSI is 100 if there are no losses
-    }
-    const firstRSI = 100 - (100 / (1 + (averageGain / averageLoss)));
-    if (isNaN(firstRSI)) {
-        console.error('Calculated RSI is NaN, returning 0');
-        return 0; // Return 0 if RSI calculation fails
-    }
-
-    const RSIs = [firstRSI];
-    // `Initial RSI for the first ${period} data points: ${firstRSI}`);
-
-    // Calculate the RSI for the rest of the data points
-    let previousAverageGain = averageGain;
-    let previousAverageLoss = averageLoss;
-
-    for (let i = period; i < prices.length - 1; i++) {
-        const change = prices[i + 1] - prices[i];
-        let gain = 0;
-        let loss = 0;
-
-        if (change > 0) {
-            gain = change;
-        } else {
-            loss = Math.abs(change);
-        }
-
-        // Calculate the new average gain and loss
-        previousAverageGain = (previousAverageGain * (period - 1) + gain) / period;
-        previousAverageLoss = (previousAverageLoss * (period - 1) + loss) / period;
-        if (previousAverageLoss === 0) {
-            console.log('No losses in the period, RSI is 100');
-            return 100; // RSI is 100 if there are no losses
-        }
-
-        // add this RSI to the list
-        const rsi = 100 - (100 / (1 + (previousAverageGain / previousAverageLoss)));
-        RSIs.push(rsi);
-    }
-
-    // Return the last calculated RSI
-    return RSIs[RSIs.length - 1];
-}
-
-// test rsi calculation on one stock
-const testRSI = async () => {
-    const symbol = 'NVDA';
-    const history = await getSymbolHistory(symbol);
-    if (!history.chart || !history.chart.result || history.chart.result.length === 0) {
-        console.error(`No data found for symbol: ${symbol}`);
-        return;
-    }
-    const prices = history.chart.result[0].indicators.quote[0].close;
-    const timestamps = history.chart.result[0].timestamp;
-    // print the last 10 prices and dates
-    const rsi = calculateRSI(prices);
-    console.log(`RSI for ${symbol}:`, rsi);
-}
-
-const testGetSector = async () => {
-    // pull for each ticker in the sp500_tickers.json file
-    const tickers = getTickersFromFile();
-    if (tickers.length === 0) {
-        console.error('No tickers found. Please ensure sp500_tickers.json exists and is populated.');
-        return;
-    }
-    // get the sector map
-    const sectorMap = getSectorMap();
-    tickers.forEach(async (ticker) => {
-        const sector = sectorMap[ticker.symbol] ? sectorMap[ticker.symbol][0] : 'Unknown';
-        console.log(`Ticker: ${ticker.symbol}, Sector: ${sector}`);
-    });
-}
-
-export const runScreener = async () => {
-    try {
-        // gt the test histories from the file
-        // const histories = fs.readFileSync('sp500_histories.json', 'utf8');
-        // const parsedHistories = JSON.parse(histories);
-        // console.log(`Loaded ${parsedHistories.length} histories from sp500_histories.json`);
-
-        // get tickers from file
-        const tickers = getTickersFromFile();
-        if (tickers.length === 0) {
-            console.error('No tickers found. Please ensure sp500_tickers.json exists and is populated.');
-            return;
-        }
-        // console.log(`Found ${tickers.length} tickers in sp500_tickers.json`);
-        // get histories for each symbol
-        const parsedHistories = await getHistoriesForEachTicker(tickers);
-        // console.log(`Fetched histories for ${parsedHistories.length} symbols.`);
-
-
-        // format the data from the histories
-        const formattedData = formatDataFromHistories(parsedHistories);
-        // console.log('Formatted data:', formattedData.slice(0, 5)); // Print first 5 entries for brevity
-
-    } catch (error) {
-        console.error('Error in main function:', error);
-    }
-}