diff --git a/assets/ar/factuality_disinformation_harmful_content/claim_detection/COVClaimDetect_GPT35_ZeroShot.py b/assets/ar/factuality_disinformation_harmful_content/claim_detection/COVClaimDetect_GPT35_ZeroShot.py deleted file mode 100644 index f2107913..00000000 --- a/assets/ar/factuality_disinformation_harmful_content/claim_detection/COVClaimDetect_GPT35_ZeroShot.py +++ /dev/null @@ -1,65 +0,0 @@ -import os - -from llmebench.datasets import CovidClaimDataset -from llmebench.models import GPTModel -from llmebench.tasks import ClaimDetectionTask - - -def config(): - return { - "dataset": CovidClaimDataset, - "dataset_args": {}, - "task": ClaimDetectionTask, - "task_args": {}, - "model": GPTModel, - "model_args": { - "api_type": "azure", - "api_version": "2023-03-15-preview", - "api_base": os.environ["AZURE_API_URL"], - "api_key": os.environ["AZURE_API_KEY"], - "engine_name": "gpt", - "max_tries": 3, - }, - "general_args": { - "data_path": "data/factuality_disinformation_harmful_content/claim_covid19/CT22_arabic_1B_claim_test_gold.tsv" - }, - } - - -def prompt(input_sample): - return { - "system_message": "You are an AI assistant that helps people find information.", - "messages": [ - { - "sender": "user", - "text": "Does this sentence contain a factual claim? Answer only by yes or no." - "\n\nsentence: " + input_sample + "label: \n", - } - ], - } - - -def post_process(response): - input_label = response["choices"][0]["text"] - input_label = input_label.replace(".", "").strip().lower() - pred_label = "" - - if ( - "yes" in input_label - or "contains a factual claim" in input_label - or "label: 1" in input_label - ): - pred_label = "1" - if ( - input_label == "no" - or "label: 0" in input_label - or "label: no" in input_label - or "not contain a factual claim" in input_label - or "doesn't contain a factual claim" in input_label - ): - pred_label = "0" - - if pred_label == "": - pred_label = None - - return pred_label diff --git a/assets/ar/factuality_disinformation_harmful_content/claim_detection/COVClaimDetect_BLOOMZ_ZeroShot.py b/assets/ar/factuality_disinformation_harmful_content/claim_detection/ClaimDetectCOVID19_BLOOMZ_ZeroShot.py similarity index 100% rename from assets/ar/factuality_disinformation_harmful_content/claim_detection/COVClaimDetect_BLOOMZ_ZeroShot.py rename to assets/ar/factuality_disinformation_harmful_content/claim_detection/ClaimDetectCOVID19_BLOOMZ_ZeroShot.py diff --git a/assets/ar/factuality_disinformation_harmful_content/claim_detection/COVClaimDetect_GPT4_FewShot.py b/assets/ar/factuality_disinformation_harmful_content/claim_detection/ClaimDetectCOVID19_GPT4_FewShot.py similarity index 100% rename from assets/ar/factuality_disinformation_harmful_content/claim_detection/COVClaimDetect_GPT4_FewShot.py rename to assets/ar/factuality_disinformation_harmful_content/claim_detection/ClaimDetectCOVID19_GPT4_FewShot.py