Merge branch 'main' into with_cites

.gitignore

@@ -9,3 +9,4 @@ test.py
 .idea
 cardiffnlp
 venv
+logs.txt

CRUDManager.py

@@ -1,6 +1,6 @@
-from DbSetup import engine, session, databaseUrl
+from DbSetup import connection, engine, session, databaseUrl
 import pandas as pd
-from sqlalchemy import desc, select
+from sqlalchemy import desc, select, sql
 from Tables import Toots
 
 def calculateSentimentCount():
@@ -11,8 +11,8 @@ def calculateSentimentCount():
                 HAVING datetime >= DATE("now","-1 day")
                 AND datetime < DATE("now")'''
     return pd.read_sql(
-        query,
+        sql.text(query),
-        databaseUrl,
+        connection,
         parse_dates=["datetime"]
     )
 
@@ -51,7 +51,7 @@ class CRUDManager():
             print(f'Could not save data to {table}!')
 
     def loadFromDatabase(self, table:str, indexColumn=None):
-        return pd.read_sql_table(table, databaseUrl, index_col=indexColumn)
+        return pd.read_sql_table(table, connection, index_col=indexColumn)
 
     def getLastToot(self):
         stmt = select(Toots.tootId).order_by(desc('datetime'))

DbSetup.py

@@ -4,6 +4,7 @@ from sqlalchemy.ext.declarative import declarative_base
 
 databaseUrl = 'sqlite:///database.db'
 engine = create_engine(databaseUrl, future=True)
+connection = engine.connect()
 session = Session(engine)
 Base = declarative_base()
 

Main.py

@@ -7,7 +7,7 @@ import matplotlib.pyplot as plt
 import matplotlib.dates as mdates
 from TootCrawler import TootCrawler
 
-locale.setlocale(locale.LC_TIME, "en_EN.UTF-8")
+locale.setlocale(locale.LC_TIME, "en_US.UTF-8")
 init_db()
 
 mastodonAccountManager = MastodonAccountManager()
@@ -84,7 +84,7 @@ axes[0].legend(pieChartlabels, loc='upper right', bbox_to_anchor=(0.9, 0.9))
 # Line chart.
 lineChart = dataframe4LineChart.plot.line(
     ax=axes[1],
-    title='Mean of all sentiments from max positive (1) to min negative (-1)'
+    title='"Mean" of sentiments, calculated from nominal values, pos(1), neu (0), neg (-1)!'
 )
 axes[1].grid(True)
 axes[1].set_xlim([date(2023, 1, 1), date(2023, 12, 31)])

TootCrawler.py

@@ -31,7 +31,10 @@ class TootCrawler():
             timelinePagination = self.mastodonInstance.fetch_previous(timelinePagination)
         for i in allTimelineResults:
             content = self.cleanhtml(i.content)
-            language = detect(content)
+            try:
+                language = detect(content)
+            except:
+                language = None
             sentiment = self.sentiTooter.analyze(language, content)
             toot = {
                 "sentiment": sentiment[0],

requirements.txt

@@ -3,9 +3,10 @@ matplotlib
 pandas
 sqlalchemy
 vader-multi
+langdetect
 numpy
 pytz
 transformers
-langdetect
+wheel
 germansentiment
 scipy