|
@@ -1,8 +1,11 @@
|
|
|
-# define tasks for UNFCCC data repository
|
|
|
-from doit import get_var
|
|
|
+"""
|
|
|
+Define the tasks for UNFCCC data repository
|
|
|
+"""
|
|
|
import os
|
|
|
import sys
|
|
|
|
|
|
+from doit import get_var
|
|
|
+
|
|
|
# TODO: task for folder mapping
|
|
|
|
|
|
# create virtual environment
|
|
@@ -20,13 +23,25 @@ import sys
|
|
|
# 'verbosity': 2,
|
|
|
# }
|
|
|
|
|
|
+
|
|
|
def task_in_venv():
|
|
|
+ """
|
|
|
+ Check if code run from virtual environment and throw an error is not.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ Nothing
|
|
|
+
|
|
|
+ """
|
|
|
+
|
|
|
def in_venv():
|
|
|
if sys.prefix == sys.base_prefix:
|
|
|
- raise ValueError("You need to run the code from the virtual environment.")
|
|
|
+ raise ValueError( # noqa: TRY003
|
|
|
+ "You need to run the code from the virtual environment."
|
|
|
+ )
|
|
|
|
|
|
return {
|
|
|
- 'actions': [in_venv],
|
|
|
+ "actions": [in_venv],
|
|
|
}
|
|
|
|
|
|
|
|
@@ -35,241 +50,319 @@ def task_set_env():
|
|
|
"""
|
|
|
Set the environment variable for the module so data is stored in the correct folders
|
|
|
"""
|
|
|
+
|
|
|
def set_root_path():
|
|
|
os.environ["UNFCCC_GHG_ROOT_PATH"] = "."
|
|
|
|
|
|
return {
|
|
|
- 'actions': [set_root_path],
|
|
|
+ "actions": [set_root_path],
|
|
|
}
|
|
|
|
|
|
|
|
|
# Task to create the mapping files which map folder names to ISO 3-letter country codes
|
|
|
read_config_folder = {
|
|
|
- "folder": get_var('folder', None),
|
|
|
+ "folder": get_var("folder", None),
|
|
|
}
|
|
|
|
|
|
+
|
|
|
def task_map_folders():
|
|
|
"""
|
|
|
Create or update the folder mapping in the given folder
|
|
|
"""
|
|
|
return {
|
|
|
- 'actions': [f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder={read_config_folder['folder']}"],
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": [
|
|
|
+ f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+ f"--folder={read_config_folder['folder']}"
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
# Tasks for getting submissions and downloading them
|
|
|
def task_update_bur():
|
|
|
- """ Update list of BUR submissions """
|
|
|
+ """Update list of BUR submissions"""
|
|
|
return {
|
|
|
- 'targets': ['downloaded_data/UNFCCC/submissions-bur.csv'],
|
|
|
- 'actions': ['datalad run -m "Fetch BUR submissions" '
|
|
|
- '-o downloaded_data/UNFCCC/submissions-bur.csv '
|
|
|
- 'python src/unfccc_ghg_data/unfccc_downloader/fetch_submissions_bur.py'],
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "targets": ["downloaded_data/UNFCCC/submissions-bur.csv"],
|
|
|
+ "actions": [
|
|
|
+ 'datalad run -m "Fetch BUR submissions" '
|
|
|
+ "-o downloaded_data/UNFCCC/submissions-bur.csv "
|
|
|
+ "python src/unfccc_ghg_data/unfccc_downloader/fetch_submissions_bur.py"
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
def task_download_bur():
|
|
|
- """ Download BUR submissions """
|
|
|
+ """Download BUR submissions"""
|
|
|
return {
|
|
|
#'file_dep': ['downloaded_data/UNFCCC/submissions-bur.csv'],
|
|
|
# deactivate file_dep fow now as it will always run fetch submissions
|
|
|
# before download
|
|
|
- 'actions': ['datalad run -m "Download BUR submissions" '
|
|
|
- '-i downloaded_data/UNFCCC/submissions-bur.csv '
|
|
|
- 'python src/unfccc_ghg_data/unfccc_downloader/download_nonannexI.py --category=BUR',
|
|
|
- f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder=downloaded_data/UNFCCC"
|
|
|
- ],
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": [
|
|
|
+ 'datalad run -m "Download BUR submissions" '
|
|
|
+ "-i downloaded_data/UNFCCC/submissions-bur.csv "
|
|
|
+ "python src/unfccc_ghg_data/unfccc_downloader/download_nonannexI.py --category=BUR",
|
|
|
+ "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+ "--folder=downloaded_data/UNFCCC",
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
def task_update_nc():
|
|
|
- """ Update list of NC submissions """
|
|
|
+ """Update list of NC submissions"""
|
|
|
return {
|
|
|
- 'targets': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
- 'actions': ['datalad run -m "Fetch NC submissions" '
|
|
|
- '-o downloaded_data/UNFCCC/submissions-nc.csv '
|
|
|
- 'python src/unfccc_ghg_data/unfccc_downloader/fetch_submissions_nc.py'],
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "targets": ["downloaded_data/UNFCCC/submissions-nc.csv"],
|
|
|
+ "actions": [
|
|
|
+ 'datalad run -m "Fetch NC submissions" '
|
|
|
+ "-o downloaded_data/UNFCCC/submissions-nc.csv "
|
|
|
+ "python src/unfccc_ghg_data/unfccc_downloader/fetch_submissions_nc.py"
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
def task_download_nc():
|
|
|
- """ Download NC submissions """
|
|
|
+ """Download NC submissions"""
|
|
|
return {
|
|
|
#'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
# deactivate file_dep fow now as it will always run fetch submissions
|
|
|
# before download
|
|
|
- 'actions': ['datalad run -m "Download NC submissions" '
|
|
|
- '-i downloaded_data/UNFCCC/submissions-nc.csv '
|
|
|
- 'python src/unfccc_ghg_data/unfccc_downloader/download_nonannexI.py --category=NC',
|
|
|
- f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder=downloaded_data/UNFCCC"
|
|
|
- ],
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": [
|
|
|
+ 'datalad run -m "Download NC submissions" '
|
|
|
+ "-i downloaded_data/UNFCCC/submissions-nc.csv "
|
|
|
+ "python src/unfccc_ghg_data/unfccc_downloader/download_nonannexI.py --category=NC",
|
|
|
+ "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+ "--folder=downloaded_data/UNFCCC",
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
+
|
|
|
# annexI data: one update call for all data types (as they are on one page)
|
|
|
# but for each year separately.
|
|
|
# downloading is per year and
|
|
|
update_aI_config = {
|
|
|
- "year": get_var('year', None),
|
|
|
- "category": get_var('category', None),
|
|
|
+ "year": get_var("year", None),
|
|
|
+ "category": get_var("category", None),
|
|
|
}
|
|
|
|
|
|
|
|
|
def task_update_annexi():
|
|
|
- """ Update list of AnnexI submissions """
|
|
|
+ """Update list of AnnexI submissions"""
|
|
|
return {
|
|
|
- 'targets': [f"downloaded_data/UNFCCC/submissions-annexI_{update_aI_config['year']}.csv"],
|
|
|
- 'actions': [f"datalad run -m 'Fetch AnnexI submissions for {update_aI_config['year']}' "
|
|
|
- "--explicit "
|
|
|
- f"-o downloaded_data/UNFCCC/submissions-annexI_{update_aI_config['year']}.csv "
|
|
|
- f"python src/unfccc_ghg_data/unfccc_downloader/fetch_submissions_annexI.py "
|
|
|
- f"--year={update_aI_config['year']}"],
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "targets": [
|
|
|
+ f"downloaded_data/UNFCCC/submissions-annexI_{update_aI_config['year']}.csv"
|
|
|
+ ],
|
|
|
+ "actions": [
|
|
|
+ f"datalad run -m 'Fetch AnnexI submissions for {update_aI_config['year']}' "
|
|
|
+ "--explicit "
|
|
|
+ f"-o downloaded_data/UNFCCC/submissions-annexI_{update_aI_config['year']}.csv "
|
|
|
+ f"python src/unfccc_ghg_data/unfccc_downloader/fetch_submissions_annexI.py "
|
|
|
+ f"--year={update_aI_config['year']}"
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
def task_download_annexi():
|
|
|
- """ Download AnnexI submissions """
|
|
|
+ """Download AnnexI submissions"""
|
|
|
+ return {
|
|
|
+ #'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
+ # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
+ # before download
|
|
|
+ "actions": [
|
|
|
+ f"datalad run -m 'Download AnnexI submissions for "
|
|
|
+ f"{update_aI_config['category']}{update_aI_config['year']}' "
|
|
|
+ f"-i downloaded_data/UNFCCC/submissions-annexI_{update_aI_config['year']}.csv "
|
|
|
+ f"python src/unfccc_ghg_data/unfccc_downloader/download_annexI.py "
|
|
|
+ f"--category={update_aI_config['category']} --year={update_aI_config['year']}",
|
|
|
+ "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+ "--folder=downloaded_data/UNFCCC",
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+# annexI data: one update call for all data types (as they are on one page)
|
|
|
+# but for each year separately.
|
|
|
+# downloading is per year and
|
|
|
+update_btr_config = {
|
|
|
+ "round": get_var("round", None),
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+def task_update_btr():
|
|
|
+ """Update list of BTR submissions"""
|
|
|
+ return {
|
|
|
+ "targets": [
|
|
|
+ f"downloaded_data/UNFCCC/submissions-BTR{update_btr_config['round']}.csv"
|
|
|
+ ],
|
|
|
+ "actions": [
|
|
|
+ f"datalad run -m 'Fetch Biannial Transparency Report submissions for "
|
|
|
+ f"BTR{update_btr_config['round']}' "
|
|
|
+ "--explicit "
|
|
|
+ f"-o downloaded_data/UNFCCC/submissions-BTR{update_btr_config['round']}.csv "
|
|
|
+ f"./venv/bin/python UNFCCC_GHG_data/UNFCCC_downloader/fetch_submissions_btr.py "
|
|
|
+ f"--round={update_btr_config['round']}"
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["setup_venv"],
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def task_download_btr():
|
|
|
+ """Download BTR submissions"""
|
|
|
return {
|
|
|
#'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
# deactivate file_dep fow now as it will always run fetch submissions
|
|
|
# before download
|
|
|
- 'actions': [f"datalad run -m 'Download AnnexI submissions for "
|
|
|
- f"{update_aI_config['category']}{update_aI_config['year']}' "
|
|
|
- f"-i downloaded_data/UNFCCC/submissions-annexI_{update_aI_config['year']}.csv "
|
|
|
- f"python src/unfccc_ghg_data/unfccc_downloader/download_annexI.py "
|
|
|
- f"--category={update_aI_config['category']} --year={update_aI_config['year']}",
|
|
|
- f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder=downloaded_data/UNFCCC"
|
|
|
- ],
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": [
|
|
|
+ f"datalad run -m 'Download BTR submissions for "
|
|
|
+ f"BTR{update_btr_config['round']}' "
|
|
|
+ f"-i downloaded_data/UNFCCC/submissions-BTR{update_btr_config['round']}.csv "
|
|
|
+ f"./venv/bin/python UNFCCC_GHG_data/UNFCCC_downloader/download_btr.py "
|
|
|
+ f"--round={update_btr_config['round']}",
|
|
|
+ "./venv/bin/python UNFCCC_GHG_data/helper/folder_mapping.py "
|
|
|
+ "--folder=downloaded_data/UNFCCC",
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["setup_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
def task_download_ndc():
|
|
|
- """ Download NDC submissions """
|
|
|
+ """Download NDC submissions"""
|
|
|
return {
|
|
|
- 'actions': ['datalad run -m "Download NDC submissions" '
|
|
|
- 'python src/unfccc_ghg_data/unfccc_downloader/download_ndc.py',
|
|
|
- f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder=downloaded_data/UNFCCC"
|
|
|
- ],
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": [
|
|
|
+ 'datalad run -m "Download NDC submissions" '
|
|
|
+ "python src/unfccc_ghg_data/unfccc_downloader/download_ndc.py",
|
|
|
+ "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+ "--folder=downloaded_data/UNFCCC",
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
# read UNFCCC submissions.
|
|
|
# datalad run is called from within the read_UNFCCC_submission.py script
|
|
|
read_config = {
|
|
|
- "country": get_var('country', None),
|
|
|
- "submission": get_var('submission', None),
|
|
|
+ "country": get_var("country", None),
|
|
|
+ "submission": get_var("submission", None),
|
|
|
}
|
|
|
|
|
|
|
|
|
# TODO: make individual task for non-UNFCCC submissions
|
|
|
def task_read_unfccc_submission():
|
|
|
- """ Read submission for a country (if code exists) (not for CRF)"""
|
|
|
+ """Read submission for a country (if code exists) (not for CRF)"""
|
|
|
return {
|
|
|
- 'actions': [f"python src/unfccc_ghg_data/unfccc_reader/read_UNFCCC_submission.py "
|
|
|
- f"--country={read_config['country']} --submission={read_config['submission']}",
|
|
|
- f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder=extracted_data/UNFCCC"
|
|
|
- ],
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": [
|
|
|
+ f"python src/unfccc_ghg_data/unfccc_reader/read_UNFCCC_submission.py "
|
|
|
+ f"--country={read_config['country']} --submission={read_config['submission']}",
|
|
|
+ "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+ "--folder=extracted_data/UNFCCC",
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
# read UNFCCC submissions.
|
|
|
# datalad run is called from within the read_UNFCCC_submission.py script
|
|
|
read_config_crf = {
|
|
|
- "country": get_var('country', None),
|
|
|
- "submission_year": get_var('submission_year', None),
|
|
|
- "submission_date": get_var('submission_date', None),
|
|
|
- "re_read": get_var('re_read', False),
|
|
|
- "countries": get_var('countries', None),
|
|
|
- "data_year": get_var('data_year', None),
|
|
|
- "totest": get_var('totest', None),
|
|
|
+ "country": get_var("country", None),
|
|
|
+ "submission_year": get_var("submission_year", None),
|
|
|
+ "submission_date": get_var("submission_date", None),
|
|
|
+ "re_read": get_var("re_read", False),
|
|
|
+ "countries": get_var("countries", None),
|
|
|
+ "data_year": get_var("data_year", None),
|
|
|
+ "totest": get_var("totest", None),
|
|
|
}
|
|
|
|
|
|
+
|
|
|
def task_read_unfccc_crf_submission():
|
|
|
- """ Read CRF submission for a country """
|
|
|
+ """Read CRF submission for a country"""
|
|
|
actions = [
|
|
|
f"python src/unfccc_ghg_data/unfccc_crf_reader"
|
|
|
f"/read_unfccc_crf_submission_datalad.py "
|
|
|
f"--country={read_config_crf['country']} "
|
|
|
f"--submission_year={read_config_crf['submission_year']} "
|
|
|
f"--submission_date={read_config_crf['submission_date']} ",
|
|
|
- f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder=extracted_data/UNFCCC"
|
|
|
- ]
|
|
|
+ "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+ "--folder=extracted_data/UNFCCC",
|
|
|
+ ]
|
|
|
if read_config_crf["re_read"] == "True":
|
|
|
actions[0] = actions[0] + " --re_read"
|
|
|
return {
|
|
|
- 'actions': actions,
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": actions,
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
def task_read_new_unfccc_crf_for_year():
|
|
|
- """ Read CRF submission for all countries for given submission year. by default only reads
|
|
|
- data not present yet. Only reads the latest updated submission for each country."""
|
|
|
- actions = [f"python src/unfccc_ghg_data/unfccc_crf_reader"
|
|
|
- f"/read_new_unfccc_crf_for_year_datalad.py "
|
|
|
- f"--submission_year={read_config_crf['submission_year']} ",
|
|
|
- f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder=extracted_data/UNFCCC"
|
|
|
- ]
|
|
|
+ """
|
|
|
+ Read CRF submission for all countries for given submission year.
|
|
|
+
|
|
|
+ By default only reads data not present yet. Only reads the latest updated
|
|
|
+ submission for each country.
|
|
|
+ """
|
|
|
+ actions = [
|
|
|
+ f"python src/unfccc_ghg_data/unfccc_crf_reader"
|
|
|
+ f"/read_new_unfccc_crf_for_year_datalad.py "
|
|
|
+ f"--submission_year={read_config_crf['submission_year']} ",
|
|
|
+ "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+ "--folder=extracted_data/UNFCCC",
|
|
|
+ ]
|
|
|
# specifying countries is currently disabled duo to problems with command line
|
|
|
# list arguments
|
|
|
- #if read_config_crf["countries"] is not None:
|
|
|
+ # if read_config_crf["countries"] is not None:
|
|
|
# actions[0] = actions[0] + f"--countries={read_config_crf['countries']} "
|
|
|
if read_config_crf["re_read"] == "True":
|
|
|
actions[0] = actions[0] + " --re_read"
|
|
|
return {
|
|
|
#'basename': "Read_CRF_year",
|
|
|
- 'actions': actions,
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": actions,
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
def task_test_read_unfccc_crf_for_year():
|
|
|
- """ Read CRF submission for all countries for given submission year. by default only reads
|
|
|
- data not present yet. Only reads the latest updated submission for each country."""
|
|
|
- actions = [f"python "
|
|
|
- f"src/unfccc_ghg_data/unfccc_crf_reader"
|
|
|
- f"/test_read_unfccc_crf_for_year.py "
|
|
|
- f"--submission_year={read_config_crf['submission_year']} "
|
|
|
- f"--country={read_config_crf['country']} "
|
|
|
- ]
|
|
|
+ """
|
|
|
+ Test CRF reading.
|
|
|
+
|
|
|
+ Test CRF with a single year only for speed and logging to extend specifications
|
|
|
+ if necessary.
|
|
|
+ """
|
|
|
+ actions = [
|
|
|
+ f"python "
|
|
|
+ f"src/unfccc_ghg_data/unfccc_crf_reader"
|
|
|
+ f"/test_read_unfccc_crf_for_year.py "
|
|
|
+ f"--submission_year={read_config_crf['submission_year']} "
|
|
|
+ f"--country={read_config_crf['country']} "
|
|
|
+ ]
|
|
|
if read_config_crf["totest"] == "True":
|
|
|
actions[0] = actions[0] + " --totest"
|
|
|
|
|
@@ -277,118 +370,133 @@ def task_test_read_unfccc_crf_for_year():
|
|
|
actions[0] = actions[0] + f"--data_year={read_config_crf['data_year']} "
|
|
|
return {
|
|
|
#'basename': "Read_CRF_year",
|
|
|
- 'actions': actions,
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": actions,
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
def task_compile_raw_unfccc_crf_for_year():
|
|
|
- """ Read CRF submission for all countries for given submission year. by default only reads
|
|
|
- data not present yet. Only reads the latest updated submission for each country."""
|
|
|
- actions = [f"python "
|
|
|
- f"src/unfccc_ghg_data/unfccc_crf_reader/crf_raw_for_year.py "
|
|
|
- f"--submission_year={read_config_crf['submission_year']} "
|
|
|
- ]
|
|
|
+ """
|
|
|
+ Collect all latest CRF submissions for a given year
|
|
|
+
|
|
|
+ Reads the latest data fromt he extracted data folder for each country.
|
|
|
+ Notifies the user if new data are available in the downloaded_data folder
|
|
|
+ which have not yet been read.
|
|
|
+
|
|
|
+ Data are saved in the datasets/UNFCCC/CRFYYYY folder.
|
|
|
+ """
|
|
|
+ actions = [
|
|
|
+ f"python "
|
|
|
+ f"src/unfccc_ghg_data/unfccc_crf_reader/crf_raw_for_year.py "
|
|
|
+ f"--submission_year={read_config_crf['submission_year']} "
|
|
|
+ ]
|
|
|
return {
|
|
|
- 'actions': actions,
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": actions,
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
+
|
|
|
# tasks for DI reader
|
|
|
# datalad run is called from within the read_unfccc_di_for_country.py script
|
|
|
read_config_di = {
|
|
|
- "country": get_var('country', None),
|
|
|
- "date": get_var('date', None),
|
|
|
- "annexI": get_var('annexI', False),
|
|
|
- #"countries": get_var('countries', None),
|
|
|
+ "country": get_var("country", None),
|
|
|
+ "date": get_var("date", None),
|
|
|
+ "annexI": get_var("annexI", False),
|
|
|
+ # "countries": get_var('countries', None),
|
|
|
}
|
|
|
|
|
|
+
|
|
|
def task_read_unfccc_di_for_country():
|
|
|
- """ Read DI data for a country """
|
|
|
+ """Read DI data for a country"""
|
|
|
actions = [
|
|
|
f"python "
|
|
|
f"src/unfccc_ghg_data/unfccc_di_reader/read_unfccc_di_for_country_datalad.py "
|
|
|
f"--country={read_config_di['country']}",
|
|
|
- f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder=extracted_data/UNFCCC"
|
|
|
- ]
|
|
|
+ "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+ "--folder=extracted_data/UNFCCC",
|
|
|
+ ]
|
|
|
return {
|
|
|
- 'actions': actions,
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": actions,
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
+
|
|
|
def task_process_unfccc_di_for_country():
|
|
|
- """ Process DI data for a country """
|
|
|
+ """Process DI data for a country"""
|
|
|
actions = [
|
|
|
f"python "
|
|
|
f"src/unfccc_ghg_data/unfccc_di_reader/process_unfccc_di_for_country_datalad"
|
|
|
f".py "
|
|
|
f"--country={read_config_di['country']} --date={read_config_di['date']}",
|
|
|
- f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder=extracted_data/UNFCCC"
|
|
|
- ]
|
|
|
+ "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+ "--folder=extracted_data/UNFCCC",
|
|
|
+ ]
|
|
|
return {
|
|
|
- 'actions': actions,
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": actions,
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
+
|
|
|
def task_read_unfccc_di_for_country_group():
|
|
|
- """ Read DI data for a country group """
|
|
|
+ """Read DI data for a country group"""
|
|
|
actions = [
|
|
|
- f"python "
|
|
|
- f"src/unfccc_ghg_data/unfccc_di_reader/read_unfccc_di_for_country_group_datalad"
|
|
|
- f".py",
|
|
|
- f"python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder=extracted_data/UNFCCC"
|
|
|
- ]
|
|
|
+ "python "
|
|
|
+ "src/unfccc_ghg_data/unfccc_di_reader/read_unfccc_di_for_country_group_datalad"
|
|
|
+ ".py",
|
|
|
+ "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+ "--folder=extracted_data/UNFCCC",
|
|
|
+ ]
|
|
|
if read_config_di["annexI"] == "True":
|
|
|
actions[0] = actions[0] + " --annexI"
|
|
|
|
|
|
return {
|
|
|
- 'actions': actions,
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": actions,
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
def task_process_unfccc_di_for_country_group():
|
|
|
- """ Process DI data for a country group """
|
|
|
+ """Process DI data for a country group"""
|
|
|
actions = [
|
|
|
- f"python "
|
|
|
- f"src/unfccc_ghg_data/unfccc_di_reader"
|
|
|
- f"/process_unfccc_di_for_country_group_datalad"
|
|
|
- f".py",
|
|
|
- ]
|
|
|
+ "python "
|
|
|
+ "src/unfccc_ghg_data/unfccc_di_reader"
|
|
|
+ "/process_unfccc_di_for_country_group_datalad"
|
|
|
+ ".py",
|
|
|
+ ]
|
|
|
if read_config_di["annexI"] == "True":
|
|
|
actions[0] = actions[0] + " --annexI"
|
|
|
if read_config_di["date"] is not None:
|
|
|
actions[0] = actions[0] + f" --date={read_config_di['date']}"
|
|
|
|
|
|
return {
|
|
|
- 'actions': actions,
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": actions,
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
+
|
|
|
# general tasks
|
|
|
def task_country_info():
|
|
|
- """ Print information on submissions and datasets
|
|
|
- available for given country"""
|
|
|
+ """
|
|
|
+ Print information on submissions and datasets available for given country
|
|
|
+ """
|
|
|
return {
|
|
|
- 'actions': [f"python src/unfccc_ghg_data/helper/country_info.py "
|
|
|
- f"--country={read_config['country']}"],
|
|
|
- 'task_dep': ['set_env'],
|
|
|
- 'verbosity': 2,
|
|
|
- 'setup': ['in_venv'],
|
|
|
+ "actions": [
|
|
|
+ f"python src/unfccc_ghg_data/helper/country_info.py "
|
|
|
+ f"--country={read_config['country']}"
|
|
|
+ ],
|
|
|
+ "task_dep": ["set_env"],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
}
|
|
|
-
|