diff --git a/.env b/.env
new file mode 100644
index 0000000..003d94a
--- /dev/null
+++ b/.env
@@ -0,0 +1 @@
+CSS_DIR="styles/main.css"
\ No newline at end of file
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..87e654b
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+*.csv filter=lfs diff=lfs merge=lfs -text
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..e710407
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,7 @@
+myenv
+venv
+.ipynb_checkpoints
+/BatchProcess/DataSource/YahooFinance/.ipynb_checkpoints
+__pycache__
+build
+dist
diff --git a/.sonarlint/connectedMode.json b/.sonarlint/connectedMode.json
new file mode 100644
index 0000000..d102997
--- /dev/null
+++ b/.sonarlint/connectedMode.json
@@ -0,0 +1,4 @@
+{
+ "sonarCloudOrganization": "nolanmm",
+ "projectKey": "NolanMM_DeepLearning_Quant_Trading_Group_Project"
+}
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..341a386
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,6 @@
+{
+ "sonarlint.connectedMode.project": {
+ "connectionId": "nolanmm",
+ "projectKey": "NolanMM_DeepLearning_Quant_Trading_Group_Project"
+ }
+}
\ No newline at end of file
diff --git a/1_HomePage.py b/1_HomePage.py
new file mode 100644
index 0000000..bc84e69
--- /dev/null
+++ b/1_HomePage.py
@@ -0,0 +1,209 @@
+from BatchProcess.DataSource.ListSnP500.ListSnP500Collect import ListSAndP500
+from BatchProcess.BatchProcess import BatchProcessManager
+from multiprocessing.pool import ThreadPool
+import plotly.graph_objects as go
+from dotenv import load_dotenv
+from pathlib import Path
+import streamlit as st
+import pandas as pd
+import time
+import os
+
+pool = ThreadPool(processes=6)
+load_dotenv(override=True)
+
+current_dir = Path(__file__).parent if "__file__" in locals() else Path.cwd()
+css_file = current_dir / os.getenv("CSS_DIR")
+defaut_start_date = "2014-01-01"
+
+st.set_page_config(page_title="Home Page", page_icon=":house:",
+ initial_sidebar_state="collapsed")
+st.sidebar.header("Quantitative Trading Project")
+st.title("Welcome to the Home Page")
+st.markdown(
+ """
+
+ """, unsafe_allow_html=True)
+
+# --- LOAD CSS ---
+with open(css_file) as f:
+ st.markdown("".format(f.read()), unsafe_allow_html=True)
+
+
+# --- CACHE DATA ---
+@st.cache_data(ttl=1800)
+def retrieve_list_ticket():
+ list_of_symbols__ = BatchProcessManager().get_stock_list_in_database()
+ if list_of_symbols__ is None or len(list_of_symbols__) < 497:
+ list_of_symbols__ = ListSAndP500().tickers_list
+ return list_of_symbols__
+
+
+@st.cache_data(ttl=1800)
+def batch_process(list_of_symbols__):
+ return BatchProcessManager().run_process(list_of_symbols__)
+
+
+@st.cache_data(ttl=1800)
+def batch_process_retrieve_data_by_stock(the_stock_in):
+ return BatchProcessManager().get_stock_data_by_ticker(the_stock_in)
+
+
+@st.cache_data
+def convert_df_to_csv(df):
+ return df.to_csv().encode("utf-8")
+
+
+@st.cache_data(ttl=1800)
+def batch_process_retrieve_all_data_in_stock_table():
+ return BatchProcessManager().get_all_stock_data_in_database()
+
+
+PROCESS_TIME = 180 # seconds
+_list_of_symbols = retrieve_list_ticket()
+
+
+# --- MAIN PAGE ---
+if "stock_data" not in st.session_state:
+ st.session_state.stock_data = None
+st.markdown('---')
+st.markdown("### I. Retrieve stock data symbol list")
+
+the_stock = st.selectbox(
+ "Select the stock you want to retrieve from database (if available)", _list_of_symbols)
+
+retrieve_col1, retrieve_col2, retrieve_col3 = st.columns(3)
+with retrieve_col1:
+ btn_prepare = st.button("Retrieve stock data from database...")
+
+# Download data by ticket button
+with retrieve_col2:
+ btn_retrieve_data_by_ticket = st.button(
+ "Process File for Ticket Data in Database (csv)")
+
+ if btn_retrieve_data_by_ticket:
+ st.session_state.stock_data = the_stock
+ df = batch_process_retrieve_data_by_stock(the_stock)
+ if df is not None:
+ df = pd.DataFrame(df)
+ csv = convert_df_to_csv(df)
+ st.download_button(
+ label="Download Ticket as CSV",
+ data=csv,
+ file_name=f"Ticket_{the_stock}_data.csv",
+ mime="text/csv",
+ )
+ else:
+ st.error(
+ "No data found for this stock, please update the database first.")
+
+# Download all data in database button
+with retrieve_col3:
+ btn_retrieve_all_data = st.button("Download All Data in Database(csv)")
+ if btn_retrieve_all_data:
+ st.session_state.stock_data = the_stock
+ df = batch_process_retrieve_all_data_in_stock_table()
+ if df is not None:
+ csv = convert_df_to_csv(df)
+ st.download_button(
+ label="Download All Data as CSV",
+ data=csv,
+ file_name="All_Stock_data.csv",
+ mime="text/csv",
+ )
+ else:
+ st.error(
+ "No data found for in database, please update the database first.")
+
+if btn_prepare:
+ st.session_state.stock_data = the_stock
+
+st.markdown('---')
+# --- TABS ---
+st.markdown(
+ "### II. List of 500 S&P, Historical data, In Day Data, Top News, Reddit News")
+List500, Historical_data, IndayData_RealTime, news, reddit_news = st.tabs(
+ ["List 500 S&P", "Historical data", "In Day Data", "Top News", "Reddit News"])
+
+# --- TABS LIST500 S&P CONTENT---
+with List500:
+ st.write("List of 500 S&P")
+ st.write(_list_of_symbols)
+
+# --- TABS HISTORICAL DATA CONTENT---
+with Historical_data:
+ if st.session_state.stock_data is not None:
+ df = batch_process_retrieve_data_by_stock(st.session_state.stock_data)
+ if df is not None:
+ df = pd.DataFrame(df)
+ fig = go.Figure(data=[go.Candlestick(x=df['date'],
+ open=df['open'],
+ high=df['high'],
+ low=df['low'],
+ close=df['close'])])
+ # Add a title
+ fig.update_layout(
+ title=f"{st.session_state.stock_data} Price Candlestick Chart",
+ # Center the title
+ title_x=0.3,
+
+ # Customize the font and size of the title
+ title_font=dict(size=24, family="Arial"),
+
+ # Set the background color of the plot
+ plot_bgcolor='white',
+
+ # Customize the grid lines
+ xaxis=dict(showgrid=True, gridwidth=1, gridcolor='lightgray'),
+ yaxis=dict(showgrid=True, gridwidth=1, gridcolor='lightgray'),
+ )
+
+ # Add a range slider and customize it
+ fig.update_layout(
+ xaxis_rangeslider_visible=True, # Show the range slider
+
+ # Customize the range slider's appearance
+ xaxis_rangeslider=dict(
+ thickness=0.1, # Set the thickness of the slider
+ bordercolor='black', # Set the border color
+ borderwidth=1, # Set the border width
+ )
+ )
+
+ # Display the chart in Streamlit
+ st.plotly_chart(fig)
+ st.markdown(
+ f"#### Dataframe of {st.session_state.stock_data} Prices")
+ st.write(df)
+ else:
+ st.write(
+ "No data found for this stock, please update the database first.")
+ else:
+ st.write("Please select the stock to retrieve the data")
+
+st.markdown('---')
+# --- Set Up/ Update all data in database---
+st.markdown("### III. Set Up data in database for the first time")
+update_database = st.button("Update Database")
+if update_database:
+ async_result = pool.apply_async(
+ batch_process, args=(_list_of_symbols,))
+ bar = st.progress(0)
+ per = PROCESS_TIME / 100
+ for i in range(100):
+ time.sleep(per)
+ bar.progress(i + 1)
+ df_dict = async_result.get()
+ st.write("Please check the data in the database")
diff --git a/BatchProcess/.env b/BatchProcess/.env
new file mode 100644
index 0000000..8ed62df
--- /dev/null
+++ b/BatchProcess/.env
@@ -0,0 +1,12 @@
+DATABASE_SERVER="localhost"
+DATABASE_PORT="5432"
+DATABASE_NAME="postgres"
+DATABASE_USER="postgres"
+DATABASE_PASSWORD="admin"
+CREATE_SCHEMA_QUERY="CREATE SCHEMA IF NOT EXISTS tickets;"
+
+INSERT_QUERY_REDDIT_TABLE="INSERT INTO reddits.stock_reddit_news (id, subreddit, url, title, score, num_comments, downvotes, ups, date_created_utc) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (id) DO NOTHING;"
+POSTGRE_CONNECTION="dbname=postgres user=postgres host=localhost password=admin"
+CONFIGURE_REDDIT_TABLE = "CREATE INDEX IF NOT EXISTS idx_stock_reddit_news_id ON reddits.stock_reddit_news(id);"
+CREATE_REDDIT_TABLE_QUERY = "CREATE TABLE IF NOT EXISTS reddits.stock_reddit_news (id VARCHAR PRIMARY KEY, subreddit VARCHAR, url VARCHAR, title TEXT, score TEXT, num_comments TEXT, downvotes TEXT, ups TEXT, date_created_utc TEXT);"
+CREATE_REDDIT_SCHEMA_QUERY="CREATE SCHEMA IF NOT EXISTS reddits;"
diff --git a/BatchProcess/BatchProcess.py b/BatchProcess/BatchProcess.py
new file mode 100644
index 0000000..2fa5315
--- /dev/null
+++ b/BatchProcess/BatchProcess.py
@@ -0,0 +1,129 @@
+from Database.PostGreSQLInteraction import DatabaseManager, StockDatabaseManager, TicketDimDatabaseManager, RedditNewsDatabaseManager
+from BatchProcess.DataSource.YahooFinance.YahooFinances_Services import YahooFinance
+from dotenv import load_dotenv
+from datetime import datetime
+import pandas as pd
+import psycopg2
+import os
+
+defaut_start_date = "2014-01-01"
+
+date_to = datetime.now().strftime('%Y-%m-%d')
+
+load_dotenv(override=True)
+
+postgres_server = os.getenv("DATABASE_SERVER")
+postgres_port = os.getenv("DATABASE_PORT")
+postgres_dbname = os.getenv("DATABASE_NAME")
+postgres_user = os.getenv("DATABASE_USER")
+postgres_pass = os.getenv("DATABASE_PASSWORD")
+
+
+class BatchProcessManager:
+ def __init__(self):
+ self.list_of_symbols = None
+ self.dict_ticket = dict()
+ self.dbname = postgres_dbname
+ self.user = postgres_user
+ self.password = postgres_pass
+ self.host = postgres_server
+ self.port = postgres_port
+ self.conn = self.create_connection()
+
+ def create_connection(self):
+ """
+ Create a connection to the database
+ """
+ try:
+ conn = psycopg2.connect(
+ dbname=self.dbname,
+ user=self.user,
+ password=self.password,
+ host=self.host,
+ port=self.port
+ )
+ return conn
+ except Exception as e:
+ print(e)
+ return None
+
+ def run_process(self, list_of_symbols_):
+ self.list_of_symbols = list_of_symbols_
+
+ # Get data from Yahoo Finance
+ transformed_data = YahooFinance(
+ self.list_of_symbols, defaut_start_date, date_to)
+ df = transformed_data.process_data()
+
+ # Create Database Manager
+ db_manager = DatabaseManager()
+
+ # Drop all tables exist in the database
+ db_manager.delete_schema()
+
+ # Create Stock table
+ db_manager.StockDatabaseManager.create_schema_and_tables(
+ self.list_of_symbols)
+
+ # Create TicketDim table
+ db_manager.TicketDimDatabaseManager.create_table()
+
+ # Create RedditNews table
+ db_manager.RedditNewsDatabaseManager.create_schema_and_tables()
+
+ # Apply multiprocessing to insert data into the database (Testing later)
+ for i in range(len(self.list_of_symbols)):
+ filtered_data = df[df['stock_id'] == self.list_of_symbols[i]]
+ filtered_data = filtered_data.reset_index()
+ self.dict_ticket[self.list_of_symbols[i]] = filtered_data
+
+ # Insert data into the database Stock table
+ for key, value in self.dict_ticket.items():
+ if isinstance(value, pd.DataFrame):
+ db_manager.StockDatabaseManager.insert_data(key, value)
+
+ # Insert data into the database TicketDim table
+ db_manager.TicketDimDatabaseManager.insert_data(self.list_of_symbols)
+
+ db_manager.close_connection()
+ return self.dict_ticket
+
+ def get_stock_data_by_ticker(self, ticker):
+ try:
+ # Create StockDatabaseManager
+ db_manager = StockDatabaseManager()
+ # Get data by table
+ data = db_manager.get_data_by_table(ticker)
+ db_manager.close_connection()
+ return data
+ except Exception as e:
+ print(e)
+ return None
+
+ def get_stock_list_in_database(self):
+ try:
+ # Create TicketDimDatabaseManager
+ db_manager = TicketDimDatabaseManager()
+ # Get data
+ data = db_manager.get_data()
+ db_manager.close_connection()
+ return data
+ except Exception as e:
+ print(e)
+ return None
+
+ def get_all_stock_data_in_database(self):
+ try:
+ db_manager = StockDatabaseManager()
+ data = db_manager.fetch_all_data()
+ db_manager.close_connection()
+ dataframes_list = [value for key, value in data.items()]
+ combined_dataframe = pd.concat(dataframes_list, ignore_index=True)
+ combined_dataframe['date'] = pd.to_datetime(
+ combined_dataframe['date'])
+
+ # Sort the combined dataframe by the 'date' column
+ return combined_dataframe.sort_values(by='date')
+ except Exception as e:
+ print(e)
+ return None
diff --git a/BatchProcess/DataSource/AlphaVantage/.env b/BatchProcess/DataSource/AlphaVantage/.env
new file mode 100644
index 0000000..57f11e5
--- /dev/null
+++ b/BatchProcess/DataSource/AlphaVantage/.env
@@ -0,0 +1,18 @@
+# Postgres Configuration
+POSTGRES_VERSION="org.postgresql:postgresql:42.2.20"
+POSTGRES_URL="jdbc:postgresql://localhost:5432/postgres"
+POSTGRES_USER="postgres"
+POSTGRES_PASSWORD="admin"
+POSTGRES_TABLE="FactPrices"
+
+FORMAT_FILE="jdbc"
+MODE="append"
+
+COLUMN_1="stock_id"
+COLUMN_2="date"
+COLUMN_3="open"
+COLUMN_4="high"
+COLUMN_5="low"
+COLUMN_6="close"
+COLUMN_7="volume"
+COLUMN_8="adjusted_close"
\ No newline at end of file
diff --git a/BatchProcess/DataSource/AlphaVantage/Retrieve Data Alpha Vantage.ipynb b/BatchProcess/DataSource/AlphaVantage/Retrieve Data Alpha Vantage.ipynb
new file mode 100644
index 0000000..43b51b7
--- /dev/null
+++ b/BatchProcess/DataSource/AlphaVantage/Retrieve Data Alpha Vantage.ipynb
@@ -0,0 +1,56 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "de71fd75-6dc9-4a45-952e-81732c3d8e26",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from alpha_vantage.timeseries import TimeSeries\n",
+ "import pandas as pd\n",
+ "from dotenv import load_dotenv\n",
+ "import os\n",
+ "\n",
+ "load_dotenv(override=True)\n",
+ "\n",
+ "api_key = os.getenv('alpha_vantage_api')\n",
+ "\n",
+ "symbol = 'AAPL'\n",
+ "\n",
+ "# Initialize the TimeSeries object with API key\n",
+ "ts = TimeSeries(key=api_key, output_format='pandas')\n",
+ "\n",
+ "# Specify the function to get daily financial data\n",
+ "# 'Time Series (Daily)' returns daily data\n",
+ "data, meta_data = ts.get_daily(symbol=symbol, outputsize='full')\n",
+ "\n",
+ "# Extract and display the relevant information\n",
+ "df = data.rename(columns={'1. open': 'Open', '2. high': 'High', '3. low': 'Low', '4. close': 'Close', '5. volume': 'Volume'})\n",
+ "df.index = pd.to_datetime(df.index)\n",
+ "#df.to_csv(f'{symbol.lower()}_data_alpha_vantage_daily.csv')"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.0"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/BatchProcess/DataSource/Binance/Untitled.ipynb b/BatchProcess/DataSource/Binance/Untitled.ipynb
new file mode 100644
index 0000000..198affe
--- /dev/null
+++ b/BatchProcess/DataSource/Binance/Untitled.ipynb
@@ -0,0 +1,141 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8013c201-9c54-4fdb-b23b-d161e7454cb6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "url = 'https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol=IBM&interval=5min&apikey=demo'\n",
+ "r = requests.get(url)\n",
+ "data = r.json()\n",
+ "\n",
+ "print(data)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "0c7b6a09-63a2-4ff0-b71f-141378b40450",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Empty DataFrame\n",
+ "Columns: [1. open, 2. high, 3. low, 4. close, 5. volume]\n",
+ "Index: []\n"
+ ]
+ }
+ ],
+ "source": [
+ "from alpha_vantage.timeseries import TimeSeries\n",
+ "import pandas as pd\n",
+ "\n",
+ "# Replace with your actual Alpha Vantage API key\n",
+ "\n",
+ "\n",
+ "ts = TimeSeries(key=api_key, output_format='pandas')\n",
+ "\n",
+ "symbol = 'AAPL' # Replace with your desired symbol\n",
+ "date_str = '2023-06-19' # Replace with your desired date\n",
+ "\n",
+ "# Fetch intraday data\n",
+ "data, meta_data = ts.get_intraday(symbol=symbol, interval='1d', outputsize='full')\n",
+ "\n",
+ "# Convert index to datetime\n",
+ "data.index = pd.to_datetime(data.index)\n",
+ "\n",
+ "# Filter data for the specific date\n",
+ "data = data[data.index.date == pd.to_datetime(date_str).date()]\n",
+ "\n",
+ "print(data)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "a028f032-c284-45a4-a637-a11489b50df3",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'Meta Data': {'1. Information': 'Daily Prices (open, high, low, close) and Volumes', '2. Symbol': 'IBM', '3. Last Refreshed': '2024-06-18', '4. Output Size': 'Compact', '5. Time Zone': 'US/Eastern'}, 'Time Series (Daily)': {'2024-06-18': {'1. open': '170.0000', '2. high': '170.7500', '3. low': '168.3800', '4. close': '170.5500', '5. volume': '3386442'}, '2024-06-17': {'1. open': '168.7600', '2. high': '169.7200', '3. low': '167.5000', '4. close': '169.5000', '5. volume': '3239815'}, '2024-06-14': {'1. open': '168.2900', '2. high': '169.4700', '3. low': '167.2300', '4. close': '169.2100', '5. volume': '2777717'}, '2024-06-13': {'1. open': '169.0100', '2. high': '169.5900', '3. low': '168.3350', '4. close': '169.1200', '5. volume': '3525717'}, '2024-06-12': {'1. open': '171.3500', '2. high': '172.4700', '3. low': '168.1010', '4. close': '169.0000', '5. volume': '3522698'}, '2024-06-11': {'1. open': '169.9800', '2. high': '170.0000', '3. low': '166.8100', '4. close': '169.3200', '5. volume': '2951251'}, '2024-06-10': {'1. open': '169.5500', '2. high': '170.7600', '3. low': '168.8800', '4. close': '170.3800', '5. volume': '3444684'}, '2024-06-07': {'1. open': '168.1800', '2. high': '171.3050', '3. low': '168.0600', '4. close': '170.0100', '5. volume': '3475495'}, '2024-06-06': {'1. open': '167.3800', '2. high': '168.4400', '3. low': '166.8000', '4. close': '168.2000', '5. volume': '2207263'}, '2024-06-05': {'1. open': '166.4100', '2. high': '167.7900', '3. low': '165.7800', '4. close': '167.3800', '5. volume': '3049377'}, '2024-06-04': {'1. open': '164.6000', '2. high': '166.4000', '3. low': '163.8800', '4. close': '165.8100', '5. volume': '2594203'}, '2024-06-03': {'1. open': '166.5400', '2. high': '166.7800', '3. low': '163.5300', '4. close': '165.2800', '5. volume': '2776058'}, '2024-05-31': {'1. open': '165.7000', '2. high': '166.9700', '3. low': '163.8400', '4. close': '166.8500', '5. volume': '4905002'}, '2024-05-30': {'1. open': '165.5600', '2. high': '166.7300', '3. low': '164.2300', '4. close': '165.6300', '5. volume': '3852963'}, '2024-05-29': {'1. open': '168.0000', '2. high': '168.6300', '3. low': '166.2100', '4. close': '167.0500', '5. volume': '4206576'}, '2024-05-28': {'1. open': '170.4400', '2. high': '171.0850', '3. low': '168.6500', '4. close': '169.6600', '5. volume': '2629645'}, '2024-05-24': {'1. open': '171.4800', '2. high': '172.0100', '3. low': '170.2100', '4. close': '170.8900', '5. volume': '2587829'}, '2024-05-23': {'1. open': '175.3900', '2. high': '175.4600', '3. low': '170.4350', '4. close': '170.6700', '5. volume': '3341335'}, '2024-05-22': {'1. open': '173.3900', '2. high': '174.9900', '3. low': '172.7600', '4. close': '173.6900', '5. volume': '3294900'}, '2024-05-21': {'1. open': '169.9400', '2. high': '174.9700', '3. low': '169.9400', '4. close': '173.4700', '5. volume': '6459800'}, '2024-05-20': {'1. open': '169.0000', '2. high': '170.1600', '3. low': '168.3800', '4. close': '169.9200', '5. volume': '2726261'}, '2024-05-17': {'1. open': '168.9700', '2. high': '169.1100', '3. low': '167.3300', '4. close': '169.0300', '5. volume': '2956387'}, '2024-05-16': {'1. open': '168.2600', '2. high': '169.6300', '3. low': '167.7900', '4. close': '168.9700', '5. volume': '3492267'}, '2024-05-15': {'1. open': '167.9400', '2. high': '168.3500', '3. low': '167.3400', '4. close': '168.2600', '5. volume': '4468823'}, '2024-05-14': {'1. open': '167.8600', '2. high': '168.1300', '3. low': '166.4800', '4. close': '167.3600', '5. volume': '2600967'}, '2024-05-13': {'1. open': '167.5000', '2. high': '168.0600', '3. low': '166.7600', '4. close': '167.5600', '5. volume': '2414859'}, '2024-05-10': {'1. open': '167.1300', '2. high': '168.0700', '3. low': '166.3200', '4. close': '167.1500', '5. volume': '2255370'}, '2024-05-09': {'1. open': '167.5000', '2. high': '167.5500', '3. low': '165.8800', '4. close': '166.2700', '5. volume': '4266616'}, '2024-05-08': {'1. open': '168.0100', '2. high': '170.2600', '3. low': '167.9000', '4. close': '169.9000', '5. volume': '3522011'}, '2024-05-07': {'1. open': '169.0000', '2. high': '169.2900', '3. low': '167.9400', '4. close': '168.3800', '5. volume': '3155260'}, '2024-05-06': {'1. open': '166.5000', '2. high': '168.6700', '3. low': '166.3800', '4. close': '168.6100', '5. volume': '4222266'}, '2024-05-03': {'1. open': '165.0000', '2. high': '166.6100', '3. low': '164.9200', '4. close': '165.7100', '5. volume': '3400405'}, '2024-05-02': {'1. open': '164.3500', '2. high': '164.8800', '3. low': '162.6200', '4. close': '164.6900', '5. volume': '3829853'}, '2024-05-01': {'1. open': '165.6900', '2. high': '166.2700', '3. low': '164.3000', '4. close': '164.4300', '5. volume': '4030960'}, '2024-04-30': {'1. open': '166.4900', '2. high': '166.7600', '3. low': '165.2605', '4. close': '166.2000', '5. volume': '6011634'}, '2024-04-29': {'1. open': '167.4000', '2. high': '168.2200', '3. low': '166.2250', '4. close': '167.4300', '5. volume': '5263342'}, '2024-04-26': {'1. open': '167.5000', '2. high': '167.8700', '3. low': '165.7300', '4. close': '167.1300', '5. volume': '8983796'}, '2024-04-25': {'1. open': '168.2000', '2. high': '172.4500', '3. low': '165.6600', '4. close': '168.9100', '5. volume': '16702150'}, '2024-04-24': {'1. open': '183.1700', '2. high': '184.2900', '3. low': '181.4000', '4. close': '184.1000', '5. volume': '7616643'}, '2024-04-23': {'1. open': '182.7300', '2. high': '184.6800', '3. low': '179.0000', '4. close': '182.1900', '5. volume': '5950229'}, '2024-04-22': {'1. open': '182.4500', '2. high': '183.3150', '3. low': '180.4500', '4. close': '181.9000', '5. volume': '3076451'}, '2024-04-19': {'1. open': '182.4300', '2. high': '182.8000', '3. low': '180.5700', '4. close': '181.5800', '5. volume': '3037990'}, '2024-04-18': {'1. open': '182.3500', '2. high': '183.4600', '3. low': '180.1700', '4. close': '181.4700', '5. volume': '2886733'}, '2024-04-17': {'1. open': '184.1600', '2. high': '184.6700', '3. low': '181.7800', '4. close': '183.1000', '5. volume': '3003033'}, '2024-04-16': {'1. open': '185.5900', '2. high': '185.7100', '3. low': '182.8600', '4. close': '183.7500', '5. volume': '4473654'}, '2024-04-15': {'1. open': '185.5700', '2. high': '187.4800', '3. low': '180.8800', '4. close': '181.2500', '5. volume': '3528140'}, '2024-04-12': {'1. open': '184.0000', '2. high': '185.1699', '3. low': '181.6850', '4. close': '182.2700', '5. volume': '3547378'}, '2024-04-11': {'1. open': '186.0400', '2. high': '186.7950', '3. low': '184.5800', '4. close': '185.9000', '5. volume': '2861736'}, '2024-04-10': {'1. open': '187.4200', '2. high': '187.9150', '3. low': '185.5200', '4. close': '186.0400', '5. volume': '3081915'}, '2024-04-09': {'1. open': '190.5400', '2. high': '191.2500', '3. low': '186.6600', '4. close': '189.3100', '5. volume': '2790673'}, '2024-04-08': {'1. open': '189.2400', '2. high': '190.2400', '3. low': '188.9118', '4. close': '189.8200', '5. volume': '2673611'}, '2024-04-05': {'1. open': '188.5900', '2. high': '190.3200', '3. low': '188.0200', '4. close': '189.1400', '5. volume': '2012428'}, '2024-04-04': {'1. open': '192.0000', '2. high': '193.2800', '3. low': '187.3400', '4. close': '187.9400', '5. volume': '2924438'}, '2024-04-03': {'1. open': '188.6000', '2. high': '191.3500', '3. low': '188.4850', '4. close': '190.9000', '5. volume': '2818910'}, '2024-04-02': {'1. open': '189.1400', '2. high': '189.8000', '3. low': '187.6000', '4. close': '188.8800', '5. volume': '2689711'}, '2024-04-01': {'1. open': '190.0000', '2. high': '190.4600', '3. low': '188.5200', '4. close': '189.8300', '5. volume': '2362586'}, '2024-03-28': {'1. open': '190.9400', '2. high': '191.9299', '3. low': '190.3400', '4. close': '190.9600', '5. volume': '3742169'}, '2024-03-27': {'1. open': '189.6000', '2. high': '190.9600', '3. low': '188.6000', '4. close': '190.8000', '5. volume': '3693305'}, '2024-03-26': {'1. open': '189.0200', '2. high': '190.0000', '3. low': '188.5000', '4. close': '188.5000', '5. volume': '4229535'}, '2024-03-25': {'1. open': '190.2600', '2. high': '190.8200', '3. low': '188.7500', '4. close': '188.7900', '5. volume': '3718289'}, '2024-03-22': {'1. open': '192.0000', '2. high': '192.9850', '3. low': '190.5100', '4. close': '190.8400', '5. volume': '3988398'}, '2024-03-21': {'1. open': '193.0000', '2. high': '193.3700', '3. low': '190.0100', '4. close': '191.9000', '5. volume': '6013561'}, '2024-03-20': {'1. open': '192.8700', '2. high': '193.9800', '3. low': '191.3100', '4. close': '193.9600', '5. volume': '3238643'}, '2024-03-19': {'1. open': '191.4900', '2. high': '193.5800', '3. low': '190.2800', '4. close': '193.3400', '5. volume': '5317341'}, '2024-03-18': {'1. open': '191.7000', '2. high': '193.2300', '3. low': '190.3200', '4. close': '191.6900', '5. volume': '5410562'}, '2024-03-15': {'1. open': '191.9900', '2. high': '193.0573', '3. low': '190.7000', '4. close': '191.0700', '5. volume': '8828184'}, '2024-03-14': {'1. open': '196.9500', '2. high': '197.7480', '3. low': '192.1200', '4. close': '193.4300', '5. volume': '4102202'}, '2024-03-13': {'1. open': '197.5500', '2. high': '198.1000', '3. low': '195.3200', '4. close': '196.7000', '5. volume': '3960737'}, '2024-03-12': {'1. open': '192.4600', '2. high': '199.1800', '3. low': '192.1500', '4. close': '197.7800', '5. volume': '5862512'}, '2024-03-11': {'1. open': '195.0900', '2. high': '195.3800', '3. low': '190.8800', '4. close': '191.7300', '5. volume': '4712688'}, '2024-03-08': {'1. open': '196.0600', '2. high': '197.7700', '3. low': '194.3800', '4. close': '195.9500', '5. volume': '3943113'}, '2024-03-07': {'1. open': '197.5800', '2. high': '198.7300', '3. low': '196.1400', '4. close': '196.5400', '5. volume': '4604458'}, '2024-03-06': {'1. open': '193.5000', '2. high': '198.1300', '3. low': '192.9600', '4. close': '196.1600', '5. volume': '6945818'}, '2024-03-05': {'1. open': '192.0000', '2. high': '193.9400', '3. low': '190.5700', '4. close': '191.9500', '5. volume': '5653641'}, '2024-03-04': {'1. open': '187.7600', '2. high': '193.8980', '3. low': '187.6000', '4. close': '193.0600', '5. volume': '7938266'}, '2024-03-01': {'1. open': '185.4900', '2. high': '188.3800', '3. low': '185.1800', '4. close': '188.2000', '5. volume': '4018354'}, '2024-02-29': {'1. open': '186.1500', '2. high': '186.8495', '3. low': '184.6900', '4. close': '185.0300', '5. volume': '6458487'}, '2024-02-28': {'1. open': '184.6300', '2. high': '185.3700', '3. low': '183.5500', '4. close': '185.3000', '5. volume': '3216345'}, '2024-02-27': {'1. open': '184.1600', '2. high': '185.1300', '3. low': '182.6200', '4. close': '184.8700', '5. volume': '3641378'}, '2024-02-26': {'1. open': '185.6000', '2. high': '186.1250', '3. low': '184.0600', '4. close': '184.1300', '5. volume': '4620815'}, '2024-02-23': {'1. open': '184.9000', '2. high': '186.4550', '3. low': '184.5700', '4. close': '185.7200', '5. volume': '3433800'}, '2024-02-22': {'1. open': '182.4500', '2. high': '184.5500', '3. low': '181.9300', '4. close': '184.2100', '5. volume': '5078398'}, '2024-02-21': {'1. open': '182.5600', '2. high': '183.0300', '3. low': '178.7500', '4. close': '179.7000', '5. volume': '4728473'}, '2024-02-20': {'1. open': '187.6400', '2. high': '188.7700', '3. low': '183.0600', '4. close': '183.4400', '5. volume': '4247181'}, '2024-02-16': {'1. open': '186.6300', '2. high': '188.9500', '3. low': '185.9452', '4. close': '187.6400', '5. volume': '4842840'}, '2024-02-15': {'1. open': '183.6200', '2. high': '186.9800', '3. low': '183.6200', '4. close': '186.8700', '5. volume': '4714301'}, '2024-02-14': {'1. open': '185.0000', '2. high': '185.0000', '3. low': '182.2600', '4. close': '183.5700', '5. volume': '3173391'}, '2024-02-13': {'1. open': '184.2800', '2. high': '184.7700', '3. low': '182.3600', '4. close': '183.7000', '5. volume': '4290453'}, '2024-02-12': {'1. open': '185.9000', '2. high': '186.4800', '3. low': '184.0300', '4. close': '186.1600', '5. volume': '4724021'}, '2024-02-09': {'1. open': '184.4400', '2. high': '187.1800', '3. low': '183.8500', '4. close': '186.3400', '5. volume': '5064641'}, '2024-02-08': {'1. open': '182.6300', '2. high': '184.5500', '3. low': '181.4900', '4. close': '184.3600', '5. volume': '5161185'}, '2024-02-07': {'1. open': '183.3400', '2. high': '184.0200', '3. low': '182.6250', '4. close': '183.7400', '5. volume': '4841188'}, '2024-02-06': {'1. open': '183.5500', '2. high': '184.6800', '3. low': '183.0400', '4. close': '183.4100', '5. volume': '3338196'}, '2024-02-05': {'1. open': '185.5100', '2. high': '185.7800', '3. low': '183.2550', '4. close': '183.4200', '5. volume': '4379602'}, '2024-02-02': {'1. open': '187.1000', '2. high': '187.3900', '3. low': '185.6150', '4. close': '185.7900', '5. volume': '4055411'}, '2024-02-01': {'1. open': '183.6300', '2. high': '187.5100', '3. low': '182.7100', '4. close': '186.9000', '5. volume': '4669444'}, '2024-01-31': {'1. open': '187.0500', '2. high': '187.6500', '3. low': '183.1400', '4. close': '183.6600', '5. volume': '8876055'}, '2024-01-30': {'1. open': '187.7100', '2. high': '188.6500', '3. low': '186.7700', '4. close': '187.8700', '5. volume': '4575058'}, '2024-01-29': {'1. open': '187.4600', '2. high': '189.4600', '3. low': '186.0500', '4. close': '187.1400', '5. volume': '6107908'}, '2024-01-26': {'1. open': '191.3100', '2. high': '192.3896', '3. low': '186.1600', '4. close': '187.4200', '5. volume': '9895941'}}}\n"
+ ]
+ }
+ ],
+ "source": [
+ "import requests\n",
+ "api_key = 'E51I8SW56CK2Z1NQ'\n",
+ "# replace the \"demo\" apikey below with your own key from https://www.alphavantage.co/support/#api-key\n",
+ "url = 'https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=IBM&apikey=E51I8SW56CK2Z1NQ'\n",
+ "r = requests.get(url)\n",
+ "data = r.json()\n",
+ "\n",
+ "print(data)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "10abdb81-2011-4363-8d03-2124b4e30166",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from alpha_vantage.timeseries import TimeSeries\n",
+ "import pandas as pd\n",
+ "\n",
+ "# Replace 'YOUR_API_KEY' with your Alpha Vantage API key\n",
+ "api_key = \"E51I8SW56CK2Z1NQ\"\n",
+ "\n",
+ "# Set the symbol for the financial instrument you're interested in (e.g., 'BTCUSDT' for Bitcoin/US Dollar)\n",
+ "symbol = 'BTCUSDT'\n",
+ "\n",
+ "# Initialize the TimeSeries object with your API key\n",
+ "ts = TimeSeries(key=api_key, output_format='pandas')\n",
+ "\n",
+ "# Specify the function to get daily financial data\n",
+ "# 'Time Series (Daily)' returns daily data\n",
+ "data, meta_data = ts.get_daily(symbol=symbol, outputsize='full')\n",
+ "\n",
+ "# Extract and display the relevant information\n",
+ "df = data.rename(columns={'1. open': 'Open', '2. high': 'High', '3. low': 'Low', '4. close': 'Close', '5. volume': 'Volume'})\n",
+ "df.index = pd.to_datetime(df.index)\n",
+ "df.to_csv(f'{symbol.lower()}_data_alpha_vantage_daily.csv')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1d7f876a-bf74-447f-bba7-2315b8ee7616",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.0"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/BatchProcess/DataSource/Binance/btcusdt_data_alpha_vantage_daily.csv b/BatchProcess/DataSource/Binance/btcusdt_data_alpha_vantage_daily.csv
new file mode 100644
index 0000000..2c62af7
--- /dev/null
+++ b/BatchProcess/DataSource/Binance/btcusdt_data_alpha_vantage_daily.csv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f13a501ee1996d32e5f9253141f46a6a295b240deef0490f04561a067ae04b6b
+size 76530
diff --git a/BatchProcess/DataSource/ListSnP500/ListSnP500Collect.py b/BatchProcess/DataSource/ListSnP500/ListSnP500Collect.py
new file mode 100644
index 0000000..7015fc1
--- /dev/null
+++ b/BatchProcess/DataSource/ListSnP500/ListSnP500Collect.py
@@ -0,0 +1,27 @@
+import pandas as pd
+
+list_remove = ['GEV', 'SOLV', 'VLTO', 'BF.B', 'BRK.B']
+
+
+class ListSAndP500:
+ def __init__(self):
+ """
+ Initialize the ListSAndP500 class
+
+ Attributes:
+ tickers_string (list): A list of stock symbols in string format
+ tickers_list (list): A list of stock symbols in list format
+
+ """
+ _tickers = pd.read_html(
+ 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')[0]
+ _tickers = _tickers.Symbol.to_list()
+ self.tickers_string = [i.replace('.', '-') for i in _tickers]
+ _tickers_list_transform_ = [
+ i if i not in list_remove else False for i in _tickers]
+ self.tickers_list = [i for i in _tickers_list_transform_ if i]
+
+
+"""
+Usage: list_of_symbols = ListSAndP500().tickers_list
+"""
diff --git a/BatchProcess/DataSource/Reddit/RedditDataSource.py b/BatchProcess/DataSource/Reddit/RedditDataSource.py
new file mode 100644
index 0000000..b0a47a7
--- /dev/null
+++ b/BatchProcess/DataSource/Reddit/RedditDataSource.py
@@ -0,0 +1,130 @@
+import psycopg2
+import praw
+from datetime import datetime
+import pandas as pd
+from time import sleep
+import os
+from dotenv import load_dotenv
+
+
+class RedditToPostgres:
+ def __init__(self):
+ load_dotenv(override=True)
+
+ self.subreddits = ["trading", "stockmarket",
+ "investing", "stocks", "wallstreetbets"]
+ self.fields = ["title", "url", "subreddit",
+ "score", "num_comments", "ups", "id"]
+
+ self.client_id = os.getenv("REDDIT_CLIENT_ID")
+ self.client_secret = os.getenv("REDDIT_CLIENT_SECRET")
+ self.username = os.getenv("REDDIT_USER")
+ self.password = os.getenv("REDDIT_PASSWORD")
+ self.user_agent = os.getenv("REDDIT_USER_AGENT")
+
+ self.postgres_url = os.getenv("POSTGRES_URL")
+ self.postgres_user = os.getenv("POSTGRES_USER")
+ self.postgres_pass = os.getenv("POSTGRES_PASSWORD")
+ self.postgres_table = os.getenv("POSTGRES_TABLE")
+ self.format_file = os.getenv("FORMAT_FILE")
+ self.mode = os.getenv("MODE")
+ self.insert_query = os.getenv("INSERT_QUERY")
+ self.configure_table = os.getenv("CONFIGURE_TABLE")
+ self.create_table_query = os.getenv("CREATE_TABLE_QUERY")
+ self.connection_os_postgres = os.getenv("POSTGRE_CONNECTION")
+
+ @staticmethod
+ def convert_unix_to_datetime(unix_timestamp):
+ try:
+ if isinstance(unix_timestamp, str):
+ if unix_timestamp.isdigit():
+ unix_timestamp = int(unix_timestamp)
+ else:
+ try:
+ datetime.strptime(unix_timestamp, '%Y-%m-%d %H:%M:%S')
+ return unix_timestamp
+ except ValueError:
+ raise ValueError("Invalid Unix timestamp format")
+ elif isinstance(unix_timestamp, int):
+ pass
+ else:
+ raise ValueError("Invalid Unix timestamp format")
+
+ return datetime.utcfromtimestamp(unix_timestamp).strftime('%Y-%m-%d %H:%M:%S')
+ except Exception as e:
+ print(f"Error: {e}")
+ return None
+
+ def init_postgresql(self):
+ connection = psycopg2.connect(self.connection_os_postgres)
+ connection.set_client_encoding('UTF8')
+ cursor = connection.cursor()
+ cursor.execute(self.create_table_query)
+ cursor.execute(self.configure_table)
+ connection.commit()
+ return connection, cursor
+
+ @staticmethod
+ def write_to_postgresql(df, cursor, connection, insert_query):
+ """Write DataFrame to PostgreSQL database."""
+ for _, row in df.iterrows():
+ date_created_utc = RedditToPostgres.convert_unix_to_datetime(
+ row["date_created_utc"])
+
+ if date_created_utc is None:
+ print(
+ f"Skipping row due to invalid timestamp: {date_created_utc}")
+ continue
+ row["date_created_utc"] = str(row["date_created_utc"])
+ data = (row["id"], row["subreddit"], row["url"], row["title"].replace(
+ '\'', ""), row["score"], row["num_comments"], row["downvotes"], row["ups"], row["date_created_utc"])
+ try:
+ cursor.execute(insert_query, data)
+ connection.commit()
+ except Exception as e:
+ print(f"Error: {e}")
+ continue
+
+ def init_reddit(self):
+ """Initialize the reddit instance"""
+ return praw.Reddit(
+ client_id=self.client_id,
+ client_secret=self.client_secret,
+ user_agent=self.user_agent,
+ username=self.username,
+ password=self.password
+ )
+
+ def reddit_stream(self, reddit, cursor, connection):
+ """Stream the data from Reddit"""
+ rows = []
+ while True:
+ recent_posts = reddit.subreddit(
+ "+".join(self.subreddits)).hot(limit=10)
+ for p in recent_posts:
+ rows_dict = {field: str(getattr(p, field))
+ for field in self.fields}
+ created_utc = str(int(getattr(p, "created_utc")))
+ rows_dict["date_created_utc"] = str(
+ datetime.fromtimestamp(int(created_utc)))
+ rows_dict["downvotes"] = str(
+ getattr(p, "ups") - getattr(p, "score"))
+ rows.append(rows_dict)
+ df = pd.DataFrame(rows)
+ with open('reddit.csv', 'a', newline='', encoding='utf-8') as f:
+ df.to_csv(f, header=f.tell() == 0, index=False)
+ if not df.empty:
+ RedditToPostgres.write_to_postgresql(
+ df, cursor, connection, self.insert_query)
+ rows.clear()
+ print(f"Finish write at {datetime.now()}")
+ sleep(65)
+
+ def main(self):
+ reddit = self.init_reddit()
+ connection, cursor = self.init_postgresql()
+ self.reddit_stream(reddit, cursor, connection)
+
+
+if __name__ == "__main__":
+ RedditToPostgres().main()
diff --git a/BatchProcess/DataSource/YahooFinance/.env b/BatchProcess/DataSource/YahooFinance/.env
new file mode 100644
index 0000000..57f11e5
--- /dev/null
+++ b/BatchProcess/DataSource/YahooFinance/.env
@@ -0,0 +1,18 @@
+# Postgres Configuration
+POSTGRES_VERSION="org.postgresql:postgresql:42.2.20"
+POSTGRES_URL="jdbc:postgresql://localhost:5432/postgres"
+POSTGRES_USER="postgres"
+POSTGRES_PASSWORD="admin"
+POSTGRES_TABLE="FactPrices"
+
+FORMAT_FILE="jdbc"
+MODE="append"
+
+COLUMN_1="stock_id"
+COLUMN_2="date"
+COLUMN_3="open"
+COLUMN_4="high"
+COLUMN_5="low"
+COLUMN_6="close"
+COLUMN_7="volume"
+COLUMN_8="adjusted_close"
\ No newline at end of file
diff --git a/BatchProcess/DataSource/YahooFinance/.ipynb_checkpoints/Workbook Yahoo Retrieve Data-checkpoint.ipynb b/BatchProcess/DataSource/YahooFinance/.ipynb_checkpoints/Workbook Yahoo Retrieve Data-checkpoint.ipynb
new file mode 100644
index 0000000..986c0f4
--- /dev/null
+++ b/BatchProcess/DataSource/YahooFinance/.ipynb_checkpoints/Workbook Yahoo Retrieve Data-checkpoint.ipynb
@@ -0,0 +1,637 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "4e86daf8-e0cb-4767-bb8a-f8cf46a10fbe",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pyspark.sql.types import StructType, StructField, StringType\n",
+ "from pyspark.sql import SparkSession\n",
+ "from dotenv import load_dotenv\n",
+ "from datetime import datetime\n",
+ "import yfinance as yf\n",
+ "import pandas as pd\n",
+ "import os\n",
+ "\n",
+ "load_dotenv(override=True)\n",
+ "\n",
+ "postgres_v = os.getenv(\"POSTGRES_VERSION\")\n",
+ "postgres_url = os.getenv(\"POSTGRES_URL\")\n",
+ "postgres_user = os.getenv(\"POSTGRES_USER\")\n",
+ "postgres_pass = os.getenv(\"POSTGRES_PASSWORD\")\n",
+ "postgres_table = os.getenv(\"POSTGRES_TABLE\")\n",
+ "format_file = os.getenv(\"FORMAT_FILE\")\n",
+ "_mode = os.getenv(\"MODE\")\n",
+ "\n",
+ "config_ = postgres_v\n",
+ "\n",
+ "column_1_name = os.getenv(\"COLUMN_1\")\n",
+ "column_2_name = os.getenv(\"COLUMN_2\")\n",
+ "column_3_name = os.getenv(\"COLUMN_3\")\n",
+ "column_4_name = os.getenv(\"COLUMN_4\")\n",
+ "column_5_name = os.getenv(\"COLUMN_5\")\n",
+ "column_6_name = os.getenv(\"COLUMN_6\")\n",
+ "column_7_name = os.getenv(\"COLUMN_7\")\n",
+ "column_8_name = os.getenv(\"COLUMN_8\")\n",
+ "\n",
+ "list_remove = ['GEV','SOLV','VLTO','BF.B','BRK.B']\n",
+ "\n",
+ "class ListSAndP500:\n",
+ " def __init__(self):\n",
+ " \"\"\"\n",
+ " Initialize the ListSAndP500 class\n",
+ "\n",
+ " Attributes:\n",
+ " tickers_string (list): A list of stock symbols in string format\n",
+ " tickers_list (list): A list of stock symbols in list format\n",
+ "\n",
+ " \"\"\"\n",
+ " _tickers = pd.read_html('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')[0]\n",
+ " _tickers = _tickers.Symbol.to_list()\n",
+ " self.tickers_string = [i.replace('.','-') for i in _tickers]\n",
+ " _tickers_list_transform_ = [i if i not in list_remove else False for i in _tickers]\n",
+ " self.tickers_list = [i for i in _tickers_list_transform_ if i]\n",
+ " \n",
+ "list_of_symbols__ = ListSAndP500().tickers_list\n",
+ "\n",
+ "class YahooFinance:\n",
+ " def __init__(self, list_of_symbols, start, end):\n",
+ " self.schema = StructType([\n",
+ " StructField(column_1_name, StringType(), True),\n",
+ " StructField(column_2_name, StringType(), True),\n",
+ " StructField(column_3_name, StringType(), True),\n",
+ " StructField(column_4_name, StringType(), True),\n",
+ " StructField(column_5_name, StringType(), True),\n",
+ " StructField(column_6_name, StringType(), True),\n",
+ " StructField(column_7_name, StringType(), True)\n",
+ " #StructField(column_8_name, StringType(), True)\n",
+ " ])\n",
+ "\n",
+ " self.symbols = list_of_symbols\n",
+ " self.interval = '1d'\n",
+ " self.start = start\n",
+ " self.end = end\n",
+ " self.results = self.process_data()\n",
+ "\n",
+ " def process_data(self):\n",
+ " \"\"\"\n",
+ " Process the historical stock data for the stock symbols\n",
+ " \"\"\"\n",
+ " data = self.get_data()\n",
+ " return self.transform_data(data)\n",
+ "\n",
+ " def get_data(self):\n",
+ " \"\"\"\n",
+ " Get historical stock data from Yahoo Finance API using yfinance library\n",
+ "\n",
+ " Returns:\n",
+ " DataFrame: A DataFrame containing historical stock data\n",
+ " \"\"\"\n",
+ " try:\n",
+ " data = yf.download(\n",
+ " self.symbols,\n",
+ " start=self.start,\n",
+ " end=self.end,\n",
+ " interval=self.interval,\n",
+ " ignore_tz=True,\n",
+ " threads=5,\n",
+ " timeout=60,\n",
+ " progress=True\n",
+ " )\n",
+ " return data\n",
+ " except Exception as e:\n",
+ " print(f\"Error downloading data: {e}\")\n",
+ " return None\n",
+ "\n",
+ " def transform_data(self, df):\n",
+ " \"\"\"\n",
+ " Transform the historical stock data into a format that can be stored in a database FactPrices table\n",
+ "\n",
+ " Args:\n",
+ " df (DataFrame): A DataFrame containing historical stock data\n",
+ "\n",
+ " Returns:\n",
+ " DataFrame: A DataFrame containing transformed historical stock data with the following columns:\n",
+ " - stock_id (str): The stock symbol\n",
+ " - date (str): The date of the stock data\n",
+ " - open (float): The opening price of the stock\n",
+ " - high (float): The highest price of the stock\n",
+ " - low (float): The lowest price of the stock\n",
+ " - close (float): The closing price of the stock\n",
+ " - volume (int): The volume of the stock\n",
+ " - adjusted_close (float): The adjusted closing price of the stock\n",
+ "\n",
+ " \"\"\"\n",
+ " # Reset the index to turn the MultiIndex into columns\n",
+ " df = df.reset_index()\n",
+ "\n",
+ " # Create a list to store transformed records\n",
+ " records = []\n",
+ "\n",
+ " # Iterate over each row and stock symbol\n",
+ " for index, row in df.iterrows():\n",
+ " date = row[('Date', '')]\n",
+ " for stock in self.symbols:\n",
+ " try:\n",
+ " record = {\n",
+ " column_1_name: stock,\n",
+ " column_2_name: date,\n",
+ " column_3_name: row[('Open', stock)],\n",
+ " column_4_name: row[('High', stock)],\n",
+ " column_5_name: row[('Low', stock)],\n",
+ " column_6_name: row[('Close', stock)],\n",
+ " column_7_name: row[('Volume', stock)]\n",
+ " #column_8_name: row[('Adj Close', stock)]\n",
+ " }\n",
+ " records.append(record)\n",
+ " except KeyError as e:\n",
+ " print(f\"KeyError: {e} for stock: {stock} on date: {date}\")\n",
+ "\n",
+ " # Convert the list of records into a DataFrame\n",
+ " return pd.DataFrame(records)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "7a0cd6a5-3629-42ae-886d-03e872242276",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "[*********************100%%**********************] 498 of 498 completed\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " stock_id date open high low close \\\n",
+ "0 MMM 2014-01-02 115.426422 116.220734 115.058525 115.493309 \n",
+ "1 AOS 2014-01-02 26.965000 26.990000 26.535000 26.660000 \n",
+ "2 ABT 2014-01-02 38.090000 38.400002 38.000000 38.230000 \n",
+ "3 ABBV 2014-01-02 52.119999 52.330002 51.520000 51.980000 \n",
+ "4 ACN 2014-01-02 81.500000 81.919998 81.089996 81.129997 \n",
+ "\n",
+ " volume \n",
+ "0 3650312.0 \n",
+ "1 1297000.0 \n",
+ "2 4967500.0 \n",
+ "3 4569100.0 \n",
+ "4 2405400.0 \n"
+ ]
+ }
+ ],
+ "source": [
+ "transformed_data = YahooFinance(list_of_symbols__, '2014-01-01', datetime.now().strftime('%Y-%m-%d')).results\n",
+ "print(transformed_data.head())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "e3613f13-de97-4dae-b6c7-2b7408806ec0",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1311732"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(transformed_data)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "id": "69e16637-f9d7-4656-80b6-5f03f6f29c38",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " stock_id | \n",
+ " date | \n",
+ " open | \n",
+ " high | \n",
+ " low | \n",
+ " close | \n",
+ " volume | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 1311581 | \n",
+ " NVR | \n",
+ " 2024-06-20 | \n",
+ " 7507.569824 | \n",
+ " 7562.299805 | \n",
+ " 7422.330078 | \n",
+ " 7514.879883 | \n",
+ " 22500.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311303 | \n",
+ " BKNG | \n",
+ " 2024-06-20 | \n",
+ " 3995.000000 | \n",
+ " 4004.399902 | \n",
+ " 3961.280029 | \n",
+ " 3972.080078 | \n",
+ " 211600.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311337 | \n",
+ " CMG | \n",
+ " 2024-06-20 | \n",
+ " 3445.580078 | \n",
+ " 3445.580078 | \n",
+ " 3188.000000 | \n",
+ " 3214.419922 | \n",
+ " 844100.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311285 | \n",
+ " AZO | \n",
+ " 2024-06-20 | \n",
+ " 2979.469971 | \n",
+ " 3029.879883 | \n",
+ " 2979.469971 | \n",
+ " 3008.229980 | \n",
+ " 287400.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311308 | \n",
+ " AVGO | \n",
+ " 2024-06-20 | \n",
+ " 1803.459961 | \n",
+ " 1804.750000 | \n",
+ " 1722.219971 | \n",
+ " 1734.560059 | \n",
+ " 6595500.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311546 | \n",
+ " MTD | \n",
+ " 2024-06-20 | \n",
+ " 1466.770020 | \n",
+ " 1470.729980 | \n",
+ " 1452.270020 | \n",
+ " 1456.589966 | \n",
+ " 119400.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311424 | \n",
+ " FICO | \n",
+ " 2024-06-20 | \n",
+ " 1428.250000 | \n",
+ " 1428.609985 | \n",
+ " 1401.670044 | \n",
+ " 1412.760010 | \n",
+ " 138300.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311680 | \n",
+ " TDG | \n",
+ " 2024-06-20 | \n",
+ " 1347.869995 | \n",
+ " 1349.000000 | \n",
+ " 1325.260010 | \n",
+ " 1329.670044 | \n",
+ " 208400.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311516 | \n",
+ " LRCX | \n",
+ " 2024-06-20 | \n",
+ " 1091.920044 | \n",
+ " 1093.390015 | \n",
+ " 1050.219971 | \n",
+ " 1061.310059 | \n",
+ " 1461200.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311583 | \n",
+ " ORLY | \n",
+ " 2024-06-20 | \n",
+ " 1059.500000 | \n",
+ " 1083.239990 | \n",
+ " 1059.500000 | \n",
+ " 1080.569946 | \n",
+ " 724300.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311628 | \n",
+ " REGN | \n",
+ " 2024-06-20 | \n",
+ " 1038.439941 | \n",
+ " 1046.290039 | \n",
+ " 1028.000000 | \n",
+ " 1044.390015 | \n",
+ " 590500.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311659 | \n",
+ " SMCI | \n",
+ " 2024-06-20 | \n",
+ " 961.289978 | \n",
+ " 1014.020020 | \n",
+ " 900.320007 | \n",
+ " 917.640015 | \n",
+ " 13347900.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311375 | \n",
+ " DECK | \n",
+ " 2024-06-20 | \n",
+ " 1008.099976 | \n",
+ " 1012.450012 | \n",
+ " 980.000000 | \n",
+ " 989.440002 | \n",
+ " 381700.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311708 | \n",
+ " GWW | \n",
+ " 2024-06-20 | \n",
+ " 921.809998 | \n",
+ " 927.710022 | \n",
+ " 913.630005 | \n",
+ " 915.289978 | \n",
+ " 186400.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311401 | \n",
+ " LLY | \n",
+ " 2024-06-20 | \n",
+ " 893.000000 | \n",
+ " 905.450012 | \n",
+ " 870.130005 | \n",
+ " 885.989990 | \n",
+ " 3357900.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311365 | \n",
+ " COST | \n",
+ " 2024-06-20 | \n",
+ " 871.309998 | \n",
+ " 873.849976 | \n",
+ " 860.559998 | \n",
+ " 862.440002 | \n",
+ " 1941100.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311511 | \n",
+ " KLAC | \n",
+ " 2024-06-20 | \n",
+ " 863.369995 | \n",
+ " 864.190002 | \n",
+ " 826.549988 | \n",
+ " 829.590027 | \n",
+ " 1281900.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311557 | \n",
+ " MPWR | \n",
+ " 2024-06-20 | \n",
+ " 846.000000 | \n",
+ " 848.450012 | \n",
+ " 819.630005 | \n",
+ " 825.330017 | \n",
+ " 649900.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311300 | \n",
+ " BLK | \n",
+ " 2024-06-20 | \n",
+ " 782.020020 | \n",
+ " 784.049988 | \n",
+ " 777.520020 | \n",
+ " 782.830017 | \n",
+ " 496800.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311409 | \n",
+ " EQIX | \n",
+ " 2024-06-20 | \n",
+ " 773.000000 | \n",
+ " 774.809998 | \n",
+ " 762.340027 | \n",
+ " 763.599976 | \n",
+ " 369600.0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " stock_id | \n",
+ " date | \n",
+ " open | \n",
+ " high | \n",
+ " low | \n",
+ " close | \n",
+ " volume | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 1311581 | \n",
+ " NVR | \n",
+ " 2024-06-20 | \n",
+ " 7507.569824 | \n",
+ " 7562.299805 | \n",
+ " 7422.330078 | \n",
+ " 7514.879883 | \n",
+ " 22500.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311303 | \n",
+ " BKNG | \n",
+ " 2024-06-20 | \n",
+ " 3995.000000 | \n",
+ " 4004.399902 | \n",
+ " 3961.280029 | \n",
+ " 3972.080078 | \n",
+ " 211600.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311337 | \n",
+ " CMG | \n",
+ " 2024-06-20 | \n",
+ " 3445.580078 | \n",
+ " 3445.580078 | \n",
+ " 3188.000000 | \n",
+ " 3214.419922 | \n",
+ " 844100.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311285 | \n",
+ " AZO | \n",
+ " 2024-06-20 | \n",
+ " 2979.469971 | \n",
+ " 3029.879883 | \n",
+ " 2979.469971 | \n",
+ " 3008.229980 | \n",
+ " 287400.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311308 | \n",
+ " AVGO | \n",
+ " 2024-06-20 | \n",
+ " 1803.459961 | \n",
+ " 1804.750000 | \n",
+ " 1722.219971 | \n",
+ " 1734.560059 | \n",
+ " 6595500.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311546 | \n",
+ " MTD | \n",
+ " 2024-06-20 | \n",
+ " 1466.770020 | \n",
+ " 1470.729980 | \n",
+ " 1452.270020 | \n",
+ " 1456.589966 | \n",
+ " 119400.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311424 | \n",
+ " FICO | \n",
+ " 2024-06-20 | \n",
+ " 1428.250000 | \n",
+ " 1428.609985 | \n",
+ " 1401.670044 | \n",
+ " 1412.760010 | \n",
+ " 138300.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311680 | \n",
+ " TDG | \n",
+ " 2024-06-20 | \n",
+ " 1347.869995 | \n",
+ " 1349.000000 | \n",
+ " 1325.260010 | \n",
+ " 1329.670044 | \n",
+ " 208400.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311516 | \n",
+ " LRCX | \n",
+ " 2024-06-20 | \n",
+ " 1091.920044 | \n",
+ " 1093.390015 | \n",
+ " 1050.219971 | \n",
+ " 1061.310059 | \n",
+ " 1461200.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311583 | \n",
+ " ORLY | \n",
+ " 2024-06-20 | \n",
+ " 1059.500000 | \n",
+ " 1083.239990 | \n",
+ " 1059.500000 | \n",
+ " 1080.569946 | \n",
+ " 724300.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311628 | \n",
+ " REGN | \n",
+ " 2024-06-20 | \n",
+ " 1038.439941 | \n",
+ " 1046.290039 | \n",
+ " 1028.000000 | \n",
+ " 1044.390015 | \n",
+ " 590500.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311659 | \n",
+ " SMCI | \n",
+ " 2024-06-20 | \n",
+ " 961.289978 | \n",
+ " 1014.020020 | \n",
+ " 900.320007 | \n",
+ " 917.640015 | \n",
+ " 13347900.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311375 | \n",
+ " DECK | \n",
+ " 2024-06-20 | \n",
+ " 1008.099976 | \n",
+ " 1012.450012 | \n",
+ " 980.000000 | \n",
+ " 989.440002 | \n",
+ " 381700.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311708 | \n",
+ " GWW | \n",
+ " 2024-06-20 | \n",
+ " 921.809998 | \n",
+ " 927.710022 | \n",
+ " 913.630005 | \n",
+ " 915.289978 | \n",
+ " 186400.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311401 | \n",
+ " LLY | \n",
+ " 2024-06-20 | \n",
+ " 893.000000 | \n",
+ " 905.450012 | \n",
+ " 870.130005 | \n",
+ " 885.989990 | \n",
+ " 3357900.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311365 | \n",
+ " COST | \n",
+ " 2024-06-20 | \n",
+ " 871.309998 | \n",
+ " 873.849976 | \n",
+ " 860.559998 | \n",
+ " 862.440002 | \n",
+ " 1941100.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311511 | \n",
+ " KLAC | \n",
+ " 2024-06-20 | \n",
+ " 863.369995 | \n",
+ " 864.190002 | \n",
+ " 826.549988 | \n",
+ " 829.590027 | \n",
+ " 1281900.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311557 | \n",
+ " MPWR | \n",
+ " 2024-06-20 | \n",
+ " 846.000000 | \n",
+ " 848.450012 | \n",
+ " 819.630005 | \n",
+ " 825.330017 | \n",
+ " 649900.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311300 | \n",
+ " BLK | \n",
+ " 2024-06-20 | \n",
+ " 782.020020 | \n",
+ " 784.049988 | \n",
+ " 777.520020 | \n",
+ " 782.830017 | \n",
+ " 496800.0 | \n",
+ "
\n",
+ " \n",
+ " | 1311409 | \n",
+ " EQIX | \n",
+ " 2024-06-20 | \n",
+ " 773.000000 | \n",
+ " 774.809998 | \n",
+ " 762.340027 | \n",
+ " 763.599976 | \n",
+ " 369600.0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
1. Create Virtual Environment (Python 3.8)
+
+* If you want to create a new virtual enviroment, you can use the following command in the terminal of the project directory:
+
+ * In Windows or Linux, you can use the following command:
+
+ ```bash
+ python -m venv venv
+ ```
+
+ * Then, you can activate the virtual enviroment by using the following command:
+
+ ```bash
+ venv\Scripts\activate
+ ```
+
+ * In MacOs, you can use the following command:
+
+ ```bash
+ python3 -m venv venv
+ ```
+
+ * Then, you can activate the virtual enviroment by using the following command:
+
+ ```bash
+ source venv/Scripts/activate
+ ```
+
+* Make sure the virtual environment needed for project is activate with corresponding project directory, you can use the following command:
+
+ * In Windows or Linux, you can use the following command:
+
+ ```bash
+ venv\Scripts\activate
+ ```
+
+ * In MacOs, you can use the following command:
+
+ ```bash
+ source venv/Scripts/activate
+ ```
+
+
+ 2. Install Dependencies
+
+Install requirements.txt: Automatically installed dependencies that needed for the project:
+
+ ```bash
+ pip install NolanMQuantTradingEnvSetUp
+ ```
+ or
+ ```bash
+ pip install -r requirements.txt
+ ```
+
+ Details
+
+### 1.Data Sources
+
+- **Reddit:** For extracting sentiment data related to stock mentions.
+- **Yahoo Finance API:** For obtaining historical and real-time stock prices and financial data.
+- **Alpha Vantage:** For additional financial data and technical indicators.
+- **Stock News API:** For news articles and headlines related to stocks.
+
+### 2.Data Processing
+
+- **Kafka:** Used as a producer to extract data from various sources.
+- **Apache Spark:** Used for data transformation and processing, both in batch and real-time.
+
+### 3.Data Storage
+
+- **PostgreSQL Database:** Centralized storage for processed data including stock prices, news, Reddit mentions, and predictions.
+
+### 4.Machine Learning
+
+- **PyTorch:** For building, training, and evaluating the LSTM deep learning model.
+
+### 5.Visualization
+
+- **Streamlit:** To visualize model predictions, trading performance, and generate interactive reports.
+
+### 6.Backup and Maintenance (Optional)
+
+- **PostgreSQL Cloud Backup:** For regular data and model backups to ensure data integrity and availability.
+- **System Monitoring:** Continuous monitoring and maintenance of the system to handle issues and improve performance
+ We applying star schema architecture database include
+
+### Central Fact Table
+
+- **DimStocks**: This is the central fact table containing information about stocks. It includes the following fields:
+ - **stock_id (PK)**: Primary Key, a unique identifier for each stock.
+ - **symbol**: The stock ticker symbol.
+ - **name**: The full name of the stock.
+ - **sector_id (FK)**: Foreign Key, links to the sector dimension.
+ - **industry_id (FK)**: Foreign Key, links to the industry dimension.
+
+### Dimension Tables
+
+1. **FactPrices**: Contains detailed pricing information for each stock.
+ - **price_id (Primary Key)**: Unique identifier for each price entry.
+ - **stock_id (FK - Stocks)**: Foreign Key, links to the central stock table.
+ - **date**: The date of the price record.
+ - **open, high, low, close**: The opening, highest, lowest, and closing prices of the stock.
+ - **volume**: The trading volume of the stock.
+ - **adjusted_close**: The adjusted closing price of the stock.
+2. **FactNews**: Contains news articles and headlines related to stocks.
+ - **news_id (Primary Key)**: Unique identifier for each news entry.
+ - **stock_id (FK - Stocks)**: Foreign Key, links to the central stock table.
+ - **title**: The title of the news article.
+ - **source**: The source of the news article.
+ - **summary**: A summary of the news article.
+ - **published_date**: The publication date of the news article.
+ - **url**: The URL link to the full news article.
+3. **FactRedditMentions**: Contains data from Reddit mentions related to stocks.
+ - **mention_id (Primary Key)**: Unique identifier for each Reddit mention.
+ - **stock_id (FK - Stocks)**: Foreign Key, links to the central stock table.
+ - **post_id**: The ID of the Reddit post.
+ - **timestamp**: The timestamp of the Reddit mention.
+ - **mentions**: The number of times the stock is mentioned in the post.
+ - **upvotes**: The number of upvotes the Reddit post received.
+ - **comments**: The number of comments on the Reddit post.
+4. **FactStocksPredict**: Contains predicted stock prices generated by the LSTM model.
+ - **prediction_id (Primary Key)**: Unique identifier for each prediction entry.
+ - **stock_id (FK - Stocks)**: Foreign Key, links to the central stock table.
+ - **prediction_date**: The date of the prediction.
+ - **predicted_open, predicted_high, predicted_low, predicted_close**: Predicted opening, highest, lowest, and closing prices of the stock.
+ - **prediction_timestamp**: The timestamp when the prediction was made.
+ - **model_version**: The version of the model used for the prediction.
+ - **confidence_level**: The confidence level of the prediction.
+
+### **Note:**
+
+- **Centralized Fact Table**: The DimStocks table is at the center, and all dimension tables are connected to it via foreign keys.
+