8000 Update gts_run_all_repos to include paths and referrer files · astrochun/github-stats-pages@704876b · GitHub
[go: up one dir, main page]
Skip to content

Commit 704876b

Browse files
astrochunastrochun
committed
Update gts_run_all_repos to include paths and referrer files
 - Add db.query_referring function - Adjust db.migrate_csv to handle Referring data
1 parent 5a753a3 commit 704876b

File tree

3 files changed

+36
-3
lines changed

3 files changed

+36
-3
lines changed

‎github_stats_pages/db.py

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ def migrate_csv(
4747
log.info(f"[yellow]Loading: {filename}")
4848
df = pd.read_csv(filename, header=None, skiprows=skip_rows, names=names)
4949
log.info(f"Size of dataframe: {len(df)}")
50+
if model.__name__ == "Referring": # Add date since this isn't included
51+
file_date = filename.name[:10]
52+
df.insert(loc=0, column="date", value=file_date)
53+
5054
if model.__name__ == "Paths":
5155
repository_names = [a.split("/")[2] for a in df["path"].values]
5256
df.insert(1, "repository_name", repository_names)
@@ -56,7 +60,12 @@ def migrate_csv(
5660
query_results = list(
5761
map(func, df["repository_name"], df["date"], df["path"])
5862
)
59-
else:
63+
elif model.__name__ == "Referring":
64+
func = partial(query_referring, engine=engine, model=model)
65+
query_results = list(
66+
map(func, df["repository_name"], df["date"], df["site"])
67+
)
68+
else: # For Clone and Traffic
6069
func = partial(query, engine=engine, model=model)
6170
query_results = list(map(func, df["repository_name"], df["date"]))
6271

@@ -125,3 +134,25 @@ def query_path(
125134
return result.one()
126135
except NoResultFound:
127136
return
137+
138+
139+
def query_referring(
140+
repository_name: str,
141+
date: str,
142+
site: str,
143+
engine: Engine,
144+
model: Union[Type[SQLModel], Referring],
145+
) -> Union[SQLModel, Referring, None]:
146+
147+
with Session(engine) as session:
148+
result = session.exec(
149+
select(model).where(
150+
model.repository_name == repository_name,
151+
model.date == date,
152+
model.site == site,
153+
)
154+
)
155+
try:
156+
return result.one()
157+
except NoResultFound:
158+
return

‎github_stats_pages/models/referring.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@ class Referring(SQLModel, table=True):
77
id: Optional[int] = Field(default=None, primary_key=True)
88
repository_name: str
99
site: str
10-
date: str
10+
date: Optional[str]
1111
total: int
1212
unique: int

‎scripts/gts_run_all_repos

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,9 @@ if __name__ == "__main__":
5959
p_cwd = Path.cwd()
6060
p_data = p_cwd / "data"
6161

62-
for datatype, model in zip(["clone", "traffic"], [db.Clone, db.Traffic]):
62+
model_names = ["clone", "traffic", "paths", "referrer"]
63+
models = [db.Clone, db.Traffic, db.Paths, db.Referring]
64+
for datatype, model in zip(model_names, models):
6365
for f in p_cwd.glob(f"????-??-??-???-???-*{datatype}-stats.csv"):
6466
db.migrate_csv(f, model, engine, skip_rows=1)
6567

0 commit comments

Comments
 (0)
0