FastAPI_google_ad_scraper/utils.py at main · 0xlearner/FastAPI_google_ad_scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
import asyncio
import os
from datetime import datetime
import pytz
import pandas as pd
import aiofiles
from aiocsv import AsyncDictReader, AsyncDictWriter
import csv

dt_obj = datetime.now(pytz.timezone("Australia/Sydney"))
time_stamp = datetime.strftime(dt_obj, "%Y-%m-%d %H:%M:%S")

temp_data = f"temp_data.csv"
clean_file = f"google_ads_urls.csv"


# def csv_reader(file_path):
#     keywords = []
#     with open(
#         file_path, mode="r", encoding="utf-8", newline=""
#     ) as file:
#         data = csv.DictReader(file)
#         for col in data:
#             keywords.append(col["keywords"])
#     return keywords


# def csv_writer(data_obj):
#     try:
#         with open(temp_data, mode="w", encoding="utf-8", newline="") as file:
#             writer = csv.DictWriter(file, fieldnames=data_obj[0].keys())
#             writer.writeheader()

#             for row in data_obj:
#                 writer.writerow(row)
#     except:
#         pass

#     try:
#         df = pd.read_csv(temp_data)
#         df.drop_duplicates(inplace=True)
#         df.to_csv(clean_file, index=False)
#     except:
#         pass

#     os.remove(temp_data)


async def csv_reader(file_path):
    keywords = []
    async with aiofiles.open(file_path, mode="r", encoding="utf-8", newline="") as file:
        data = AsyncDictReader(file)
        async for col in data:
            keywords.append(col["keywords"])
    return keywords


async def csv_writer(data_obj):
    try:
        async with aiofiles.open(
            temp_data, mode="w", encoding="utf-8", newline=""
        ) as file:
            writer = AsyncDictWriter(file, fieldnames=data_obj[0].keys())
            await writer.writeheader()

            for row in data_obj:
                await writer.writerow(row)
    except:
        pass

    try:
        df = pd.read_csv(temp_data)
        df.drop_duplicates(inplace=True)
        df.to_csv(clean_file, index=False)
    except:
        pass

    os.remove(temp_data)


if __name__ == "__main__":
    # file_path = os.path.abspath("keywords_for_scraper.csv")
    # kw = csv_reader(file_path)
    data = [{"Ad_url": "www.google.com"}]
    loop = asyncio.get_event_loop()
    loop.run_until_complete(csv_writer(data))