Merge branch 'master' into 'main'

initial commit

See merge request !1

main.py

 import snscrape.modules.twitter as sntwitter
 import pandas as pd
-import datetime as dt
 
-# cool
-df = pd.DataFrame(columns=['date', 'tweet'])
-limit = 100
+from src.functions import construct_query, convert_date
 
-for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query=" Metaverse -filter:replies -filter:links -filter:retweets -filter:nativeretweets").get_items()):
-    if i > limit:
+table = pd.DataFrame(columns=['date', 'tweet'])
+
+terms = ["Metaverse"]
+negativeTerms = ["#metaverse"]
+
+filters = []
+negativeFilters = ["replies", "links", "retweets", "nativeretweets"]
+
+languages = ["en"]
+
+tweetNumber = 1000
+
+for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query=construct_query(terms, negativeTerms, filters, negativeFilters)).get_items()):
+    if i > tweetNumber:
         break
-    if (tweet.lang == "en"):
-        df2 = {'date': tweet.date.strftime(
-            "%m/%d/%Y, %H:%M:%S"), 'tweet': tweet.rawContent}
+    if (tweet.lang in languages):
+        dict = {'date': convert_date(tweet.date), 'tweet': tweet.rawContent}
+
+        if ("metaverse" in tweet.rawContent.lower()):
+            table = pd.concat([table, pd.DataFrame.from_records([dict])])
 
-        df = pd.concat([df, pd.DataFrame.from_records([df2])])
+    print(i/tweetNumber*100, " %")
 
-    print(i/limit*100)
 
+table = table.drop_duplicates(subset=["tweet"]).sort_values(by=["tweet"])
 
-df = df.drop_duplicates(subset=["tweet"])
-df = df.sort_values(by=["tweet"])
-df.to_json("twitter_data.json", orient="split")
+table.to_json("output/twitter_data.json", orient="split")

src/functions.py

+import datetime as dt
+
+
+def construct_query(terms, negativeTerms, filters, negativeFilters):
+    query = ""
+
+    for t in terms:
+        query += t+" "
+
+    for nt in negativeTerms:
+        query += "-"+nt+" "
+
+    for f in filters:
+        query += "filter:"+f+" "
+
+    for nf in negativeFilters:
+        query += "-filter:"+nf+" "
+
+    return (query)
+
+
+def convert_date(date):
+    return (date.strftime(
+            "%m/%d/%Y, %H:%M:%S"))