From a9322641ae21b634cb2f345bd46a29539cd3e20f Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 25 Aug 2024 21:28:38 -0300 Subject: [PATCH 1/7] Cria get_api() e refatora get_api_data() --- publication/api/publication.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/publication/api/publication.py b/publication/api/publication.py index 3afea2fb..5ad13e1c 100644 --- a/publication/api/publication.py +++ b/publication/api/publication.py @@ -14,28 +14,31 @@ from publication.api import exceptions -def get_api_data(collection, content_type, website_kind=None): - website_kind = website_kind or collection_choices.QA - - try: - website = WebSiteConfiguration.get( - collection=collection, - purpose=website_kind, - ) - except WebSiteConfiguration.DoesNotExist: - return {"error": f"Website does not exist: {collection} {website_kind}"} +def get_api(collection, content_type, website_kind=None): + website = WebSiteConfiguration.get( + collection=collection, + purpose=website_kind, + ) API_URLS = { "journal": website.api_url_journal, "issue": website.api_url_issue, "article": website.api_url_article, } - api = PublicationAPI( + return PublicationAPI( post_data_url=API_URLS.get(content_type), get_token_url=website.api_get_token_url, username=website.api_username, password=website.api_password, ) + + +def get_api_data(collection, content_type, website_kind=None): + try: + api = get_api(collection, content_type, website_kind) + except WebSiteConfiguration.DoesNotExist: + return {"error": f"Website does not exist: {collection} {website_kind}"} + api.get_token() return api.data From 424eda240b02cbda539a42fef4ae915895ed55bc Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 25 Aug 2024 21:28:38 -0300 Subject: [PATCH 2/7] =?UTF-8?q?Adiciona=20logging.info=20para=20a=20respos?= =?UTF-8?q?ta=20da=20execu=C3=A7=C3=A3o=20de=20publish=5Fissue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- publication/api/issue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/publication/api/issue.py b/publication/api/issue.py index e27b418c..0ac3e6e5 100644 --- a/publication/api/issue.py +++ b/publication/api/issue.py @@ -15,7 +15,9 @@ def publish_issue(issue_proc, api_data): builder = IssuePayload(data) build_issue(builder, journal_id, issue, issue_id) api = PublicationAPI(**api_data) - return api.post_data(data, {"journal_id": journal_id}) + response = api.post_data(data, {"journal_id": journal_id}) + logging.info(response) + return response class IssuePayload: From 3bfe7934257d748b2400b3fca5604f8f7de6871f Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 25 Aug 2024 21:28:38 -0300 Subject: [PATCH 3/7] =?UTF-8?q?Adiciona=20valor=20alternativo=20quando=20v?= =?UTF-8?q?alor=20de=20journal.logo=5Furl=20=C3=A9=20None?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- publication/utils/journal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publication/utils/journal.py b/publication/utils/journal.py index 341d62d2..9d2a3732 100644 --- a/publication/utils/journal.py +++ b/publication/utils/journal.py @@ -42,7 +42,7 @@ def build_journal(builder, journal, journal_id, journal_acron, journal_history): ) try: # FIXME - builder.add_logo_url(journal.logo_url) + builder.add_logo_url(journal.logo_url or "https://www.scielo.org/journal_logo_missing.gif") except AttributeError: builder.add_logo_url("https://www.scielo.org/journal_logo_missing.gif") builder.add_online_submission_url(journal.submission_online_url) # Adicionar From 88fa15e7387270264410c9120928e637682330fb Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 25 Aug 2024 21:28:38 -0300 Subject: [PATCH 4/7] =?UTF-8?q?Adiciona=20logging.info=20para=20a=20respos?= =?UTF-8?q?ta=20da=20execu=C3=A7=C3=A3o=20de=20publish=5Fjournal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- publication/api/journal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/publication/api/journal.py b/publication/api/journal.py index 2c292361..6f5718f5 100644 --- a/publication/api/journal.py +++ b/publication/api/journal.py @@ -26,8 +26,10 @@ def publish_journal(journal_proc, api_data): ) api = PublicationAPI(**api_data) + response = api.post_data(payload) logging.info(payload) - return api.post_data(payload) + logging.info(response) + return response class JournalPayload: From c5040e2b83c8dd3d0cb5e04d356fa840e3111d16 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 25 Aug 2024 21:28:38 -0300 Subject: [PATCH 5/7] =?UTF-8?q?Adiciona=20o=20par=C3=A2metro=20website=5Fk?= =?UTF-8?q?ind=20nas=20chamadas=20de=20proc.publish()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proc/controller.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proc/controller.py b/proc/controller.py index f46f5c59..be207254 100644 --- a/proc/controller.py +++ b/proc/controller.py @@ -71,6 +71,7 @@ def migrate_and_publish_journals( journal_proc.publish( user, publish_journal, + website_kind=QA, api_data=api_data, force_update=force_update, ) @@ -134,6 +135,7 @@ def migrate_and_publish_issues( issue_proc.publish( user, publish_issue, + website_kind=QA, api_data=api_data, force_update=force_update, ) From 2d754b9d50cf6fde8d6f4136387cd1c757b2743d Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 25 Aug 2024 21:28:38 -0300 Subject: [PATCH 6/7] Troca article_api_data por qa_article_api_data e public_article_api_data e adiciona tarefas para publicar journals, issues, articles --- proc/tasks.py | 365 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 348 insertions(+), 17 deletions(-) diff --git a/proc/tasks.py b/proc/tasks.py index 0a0f477a..3fee2753 100644 --- a/proc/tasks.py +++ b/proc/tasks.py @@ -13,7 +13,9 @@ from proc.controller import migrate_and_publish_issues, migrate_and_publish_journals from proc.models import ArticleProc, IssueProc, JournalProc from publication.api.document import publish_article -from publication.api.publication import get_api_data +from publication.api.journal import publish_journal +from publication.api.issue import publish_issue +from publication.api.publication import get_api_data, get_api from tracker import choices as tracker_choices from tracker.models import UnexpectedEvent @@ -69,8 +71,6 @@ def task_migrate_and_publish( ): try: from_datetime = datetime.utcnow() - user = _get_user(user_id, username) - for collection in _get_collections(collection_acron): # obtém os dados do site clássico classic_website = controller.get_classic_website(collection.acron) @@ -94,14 +94,17 @@ def task_migrate_and_publish( ) # migra os documentos - article_api_data = get_api_data(collection, "article") + qa_article_api_data = get_api_data(collection, "article", QA) + public_article_api_data = get_api_data(collection, "article", PUBLIC) + for journal_proc in JournalProc.journals_with_modified_articles(collection): task_migrate_journal_articles.apply_async( kwargs=dict( user_id=user_id, username=username, journal_proc_id=journal_proc.id, - article_api_data=article_api_data, + qa_article_api_data=qa_article_api_data, + public_article_api_data=public_article_api_data ) ) @@ -129,7 +132,8 @@ def task_migrate_journal_articles( publication_year=None, issue_folder=None, force_update=None, - article_api_data=None, + qa_article_api_data=None, + public_article_api_data=None ): """ Migra todos ou uma seleção de artigos de um dado journal @@ -188,7 +192,8 @@ def task_migrate_journal_articles( username=username, issue_proc_id=issue_proc.id, force_update=force_update, - article_api_data=article_api_data, + qa_article_api_data=qa_article_api_data, + public_article_api_data=public_article_api_data ) ) @@ -209,7 +214,8 @@ def task_migrate_journal_articles( username=username, issue_proc_id=issue_proc.id, force_update=force_update, - article_api_data=article_api_data, + qa_article_api_data=qa_article_api_data, + public_article_api_data=public_article_api_data ) ) @@ -235,7 +241,8 @@ def task_migrate_issue_articles( username=None, issue_proc_id=None, force_update=None, - article_api_data=None, + qa_article_api_data=None, + public_article_api_data=None ): try: user = _get_user(user_id, username) @@ -251,7 +258,8 @@ def task_migrate_issue_articles( username=username, article_proc_id=article_proc.id, force_update=force_update, - article_api_data=article_api_data, + qa_article_api_data=qa_article_api_data, + public_article_api_data=public_article_api_data, ) ) except Exception as e: @@ -276,7 +284,8 @@ def task_migrate_and_publish_article( username=None, article_proc_id=None, force_update=None, - article_api_data=None, + qa_article_api_data=None, + public_article_api_data=None, ): try: user = _get_user(user_id, username) @@ -287,14 +296,15 @@ def task_migrate_and_publish_article( article_proc.publish( user, publish_article, - api_data=article_api_data, + website_kind=QA, + api_data=qa_article_api_data, force_update=force_update, ) article_proc.publish( user, publish_article, website_kind=PUBLIC, - api_data=article_api_data, + api_data=public_article_api_data, force_update=force_update, ) except Exception as e: @@ -349,6 +359,109 @@ def task_migrate_and_publish_journals( ) +@celery_app.task(bind=True) +def task_publish_journals( + self, + user_id=None, + username=None, + collection_acron=None, + journal_acron=None, + force_update=False, +): + try: + user = _get_user(user_id, username) + for collection in _get_collections(collection_acron): + for website_kind in (QA, PUBLIC): + try: + api = get_api(collection, "journal", website_kind) + except WebSiteConfiguration.DoesNotExist: + continue + api.get_token() + api_data = api.data + + # FIXME + params = {} + params["collection"] = collection + if journal_acron: + params["acron"] = journal_acron + + if not force_update: + if website_kind == QA: + params["qa_ws_status__in"] = [ + tracker_choices.PROGRESS_STATUS_TODO, + tracker_choices.PROGRESS_STATUS_REPROC, + ] + elif website_kind == PUBLIC: + params["public_ws_status__in"] = [ + tracker_choices.PROGRESS_STATUS_TODO, + tracker_choices.PROGRESS_STATUS_REPROC, + ] + + items = JournalProc.objects.filter(**params) + + for journal_proc in items: + task_publish_journal.apply_async( + kwargs=dict( + user_id=user_id, + username=username, + website_kind=website_kind, + journal_proc_id=journal_proc.id, + api_data=api_data, + force_update=force_update, + ) + ) + + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "proc.tasks.task_publish_journals", + "user_id": user_id, + "username": username, + "collection_acron": collection_acron, + "force_update": force_update, + }, + ) + + +@celery_app.task(bind=True) +def task_publish_journal( + self, + user_id=None, + username=None, + website_kind=None, + journal_proc_id=None, + api_data=None, + force_update=None, +): + try: + user = _get_user(user_id, username) + journal_proc = JournalProc.objects.get(pk=journal_proc_id) + journal_proc.publish( + user, + publish_journal, + website_kind=website_kind, + api_data=api_data, + force_update=force_update, + ) + + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "proc.controller.publish_journal", + "user_id": user.id, + "username": user.username, + "website_kind": website_kind, + "pid": journal_proc.pid, + }, + ) + + ############################################ @celery_app.task(bind=True) def task_migrate_and_publish_issues( @@ -361,7 +474,6 @@ def task_migrate_and_publish_issues( ): try: user = _get_user(user_id, username) - for collection in _get_collections(collection_acron): # obtém os dados do site clássico classic_website = controller.get_classic_website(collection.acron) @@ -386,6 +498,113 @@ def task_migrate_and_publish_issues( ) +@celery_app.task(bind=True) +def task_publish_issues( + self, + user_id=None, + username=None, + collection_acron=None, + journal_acron=None, + publication_year=None, + force_update=False, +): + try: + for collection in _get_collections(collection_acron): + for website_kind in (QA, PUBLIC): + try: + api = get_api(collection, "issue", website_kind) + except WebSiteConfiguration.DoesNotExist: + continue + api.get_token() + api_data = api.data + + # FIXME + params = {} + params["collection"] = collection + if journal_acron: + params["journal_proc__acron"] = journal_acron + if publication_year: + params["issue__publication_year"] = str(publication_year) + + if not force_update: + if website_kind == QA: + params["qa_ws_status__in"] = [ + tracker_choices.PROGRESS_STATUS_TODO, + tracker_choices.PROGRESS_STATUS_REPROC, + ] + elif website_kind == PUBLIC: + params["public_ws_status__in"] = [ + tracker_choices.PROGRESS_STATUS_TODO, + tracker_choices.PROGRESS_STATUS_REPROC, + ] + + items = IssueProc.objects.filter(**params) + + for issue_proc in items: + task_publish_issue.apply_async( + kwargs=dict( + user_id=user_id, + username=username, + website_kind=website_kind, + issue_proc_id=issue_proc.id, + api_data=api_data, + force_update=force_update, + ) + ) + + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "proc.tasks.task_publish_issues", + "user_id": user_id, + "username": username, + "collection_acron": collection_acron, + "journal_acron": journal_acron, + "publication_year": publication_year, + "force_update": force_update, + }, + ) + + +@celery_app.task(bind=True) +def task_publish_issue( + self, + user_id=None, + username=None, + website_kind=None, + issue_proc_id=None, + api_data=None, + force_update=None +): + try: + user = _get_user(user_id, username) + issue_proc = IssueProc.objects.get(pk=issue_proc_id) + issue_proc.publish( + user, + publish_issue, + website_kind=website_kind, + api_data=api_data, + force_update=force_update, + ) + + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "proc.controller.publish_issue", + "user_id": user.id, + "username": user.username, + "website_kind": website_kind, + "pid": issue_proc.pid, + }, + ) + + ############################################ @celery_app.task(bind=True) def task_migrate_and_publish_articles( @@ -407,11 +626,15 @@ def task_migrate_and_publish_articles( if journal_acron: params["journal__acron"] = journal_acron + logging.info(params) article_api_data = None for journal_proc in JournalProc.objects.filter(**params): - article_api_data = article_api_data or get_api_data( - journal_proc.collection, "article" + qa_article_api_data = article_api_data or get_api_data( + journal_proc.collection, "article", QA + ) + public_article_api_data = article_api_data or get_api_data( + journal_proc.collection, "article", PUBLIC ) # como é custoso obter os registros de acron, # somente se force_import é True, reexecuta a leitura de acron.id @@ -428,7 +651,8 @@ def task_migrate_and_publish_articles( publication_year=publication_year, issue_folder=issue_folder, force_update=force_update or force_import, - article_api_data=article_api_data, + qa_article_api_data=qa_article_api_data, + public_article_api_data=public_article_api_data, # from_datetime=from_datetime, ) ) @@ -446,3 +670,110 @@ def task_migrate_and_publish_articles( "force_update": force_update, }, ) + + +@celery_app.task(bind=True) +def task_publish_articles( + self, + user_id=None, + username=None, + collection_acron=None, + journal_acron=None, + publication_year=None, + force_update=False, +): + try: + for collection in _get_collections(collection_acron): + for website_kind in (QA, PUBLIC): + try: + api = get_api(collection, "article", website_kind) + except WebSiteConfiguration.DoesNotExist: + continue + api.get_token() + api_data = api.data + + # FIXME + params = {} + params["collection"] = collection + if journal_acron: + params["issue_proc__journal_proc__acron"] = journal_acron + if publication_year: + params["issue_proc__issue__publication_year"] = str(publication_year) + + if not force_update: + if website_kind == QA: + params["qa_ws_status__in"] = [ + tracker_choices.PROGRESS_STATUS_TODO, + tracker_choices.PROGRESS_STATUS_REPROC, + ] + elif website_kind == PUBLIC: + params["public_ws_status__in"] = [ + tracker_choices.PROGRESS_STATUS_TODO, + tracker_choices.PROGRESS_STATUS_REPROC, + ] + + items = articleProc.objects.filter(**params) + + for article_proc in items: + task_publish_article.apply_async( + kwargs=dict( + user_id=user_id, + username=username, + website_kind=website_kind, + article_proc_id=article_proc.id, + api_data=api_data, + force_update=force_update, + ) + ) + + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "proc.tasks.task_publish_articles", + "user_id": user_id, + "username": username, + "collection_acron": collection_acron, + "journal_acron": journal_acron, + "publication_year": publication_year, + "force_update": force_update, + }, + ) + + +@celery_app.task(bind=True) +def task_publish_article( + self, + user_id=None, + username=None, + website_kind=None, + article_proc_id=None, + api_data=None, + force_update=None, +): + try: + user = _get_user(user_id, username) + article_proc = articleProc.objects.get(pk=article_proc_id) + article_proc.publish( + user, + publish_article, + website_kind=website_kind, + api_data=api_data, + force_update=force_update, + ) + + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + e=e, + exc_traceback=exc_traceback, + detail={ + "task": "proc.controller.publish_article", + "user_id": user_id, + "username": username, + "website_kind": website_kind, + "pid": article_proc.pid, + }, + ) From baa9088a33a28d6d671e9a955c70cc9e71eab3c3 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sun, 25 Aug 2024 21:28:38 -0300 Subject: [PATCH 7/7] =?UTF-8?q?Agenda=20as=20tarefas=20de=20publica=C3=A7?= =?UTF-8?q?=C3=A3o=20de=20journals,=20issues,=20articles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bigbang/tasks_scheduler.py | 74 +++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/bigbang/tasks_scheduler.py b/bigbang/tasks_scheduler.py index 70ac807a..47e03007 100644 --- a/bigbang/tasks_scheduler.py +++ b/bigbang/tasks_scheduler.py @@ -89,7 +89,9 @@ def schedule_migration_subtasks(username): def schedule_publication_subtasks(username): - pass + _schedule_publish_articles(username) + _schedule_publish_issues(username) + _schedule_publish_journals(username) def _schedule_migration_and_publication(username, enabled): @@ -203,3 +205,73 @@ def _schedule_migrate_and_publish_articles(username, enabled): # hour="*", # minute=SINCHRONIZE_TO_PID_PROVIDER_MINUTES, # ) + + +def _schedule_publish_journals(username, enabled=False): + """ + Agenda a tarefa de migrar os registros da base de dados TITLE + Deixa a tarefa desabilitada + """ + schedule_task( + task="proc.tasks.task_publish_journals", + name="publish_journals", + kwargs=dict( + username=None, + collection_acron=None, + journal_acron=None, + force_update=False, + ), + description=_("Publica periódicos"), + priority=TITLE_DB_MIGRATION_PRIORITY, + enabled=enabled, + run_once=False, + day_of_week="*", + hour="*", + minute=TITLE_DB_MIGRATION_MINUTES, + ) + + +def _schedule_publish_issues(username, enabled=False): + """ + Agenda a tarefa de migrar os registros da base de dados ISSUE + Deixa a tarefa abilitada + """ + schedule_task( + task="proc.tasks.task_publish_issues", + name="publish_issues", + kwargs=dict( + username=None, + collection_acron=None, + journal_acron=None, + publication_year=None, + force_update=False, + ), + description=_("Publica fascículos"), + priority=ISSUE_DB_MIGRATION_PRIORITY, + enabled=enabled, + run_once=False, + day_of_week="*", + hour="*", + minute=ISSUE_DB_MIGRATION_MINUTES, + ) + + +def _schedule_publish_articles(username, enabled=False): + schedule_task( + task="proc.tasks.task_publish_articles", + name="publish_articles", + kwargs=dict( + username=None, + collection_acron=None, + journal_acron=None, + publication_year=None, + force_update=False, + ), + description=_("Publica artigos"), + priority=ARTICLE_DB_MIGRATION_PRIORITY, + enabled=enabled, + run_once=False, + day_of_week="*", + hour="*", + minute=ARTICLE_DB_MIGRATION_MINUTES, + ) \ No newline at end of file