From 01f08fd88d5205930635c7920541f2171139192a Mon Sep 17 00:00:00 2001 From: Daoud Clarke Date: Sat, 14 Jan 2023 19:15:26 +0000 Subject: [PATCH] Return updated URLs --- mwmbl/crawler/urls.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.py index 4dc0db7..b2eb786 100644 --- a/mwmbl/crawler/urls.py +++ b/mwmbl/crawler/urls.py @@ -78,7 +78,7 @@ class URLDatabase: cursor.execute(index_sql) cursor.execute(view_sql) - def update_found_urls(self, found_urls: list[FoundURL]): + def update_found_urls(self, found_urls: list[FoundURL]) -> list[FoundURL]: if len(found_urls) == 0: return @@ -104,6 +104,7 @@ class URLDatabase: updated = CASE WHEN urls.status > excluded.status THEN urls.updated ELSE excluded.updated END + RETURNING (url, user_id_hash, score, status, timestamp) """ input_urls = [x.url for x in found_urls] @@ -129,6 +130,9 @@ class URLDatabase: for found_url in sorted_urls if found_url.url in urls_to_insert] execute_values(cursor, insert_sql, data) + results = cursor.fetchall() + updated = [FoundURL(*result) for result in results] + return updated def get_urls_for_crawling(self): start = datetime.utcnow()