|
|
@@ -8,6 +8,8 @@ import datalad.api
|
|
|
from doit import get_var
|
|
|
|
|
|
root_path = "."
|
|
|
+os.environ["UNFCCC_GHG_ROOT_PATH"] = root_path
|
|
|
+
|
|
|
|
|
|
# TODO: task for folder mapping
|
|
|
|
|
|
@@ -91,18 +93,7 @@ def task_map_folders():
|
|
|
Create or update the folder mapping in the given folder
|
|
|
"""
|
|
|
return {
|
|
|
- "actions": [
|
|
|
- set_root_path(),
|
|
|
- datalad.api.run(
|
|
|
- cmd="python3 src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- f"--folder={read_config_folder['folder']}",
|
|
|
- dataset=root_path,
|
|
|
- message=f"Update folder mapping for {read_config_folder['folder']}",
|
|
|
- outputs=f"{read_config_folder['folder']}/folder_mapping.json",
|
|
|
- dry_run=None,
|
|
|
- explicit=True,
|
|
|
- ),
|
|
|
- ],
|
|
|
+ "actions": [(map_folders, [read_config_folder["folder"]])],
|
|
|
"verbosity": 2,
|
|
|
"setup": ["in_venv"],
|
|
|
}
|
|
|
@@ -111,6 +102,18 @@ def task_map_folders():
|
|
|
# Tasks for getting submissions and downloading them
|
|
|
def task_update_bur():
|
|
|
"""Update list of BUR submissions"""
|
|
|
+
|
|
|
+ def fetch_bur():
|
|
|
+ datalad.api.run(
|
|
|
+ cmd="python3 src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
+ "fetch_submissions_bur.py",
|
|
|
+ dataset=root_path,
|
|
|
+ message="Fetch BUR submissions",
|
|
|
+ outputs="downloaded_data/UNFCCC/submissions-bur.csv",
|
|
|
+ dry_run=True,
|
|
|
+ explicit=True,
|
|
|
+ )
|
|
|
+
|
|
|
return {
|
|
|
"targets": ["downloaded_data/UNFCCC/submissions-bur.csv"],
|
|
|
# "actions": [
|
|
|
@@ -119,16 +122,7 @@ def task_update_bur():
|
|
|
# "python src/unfccc_ghg_data/unfccc_downloader/fetch_submissions_bur.py"
|
|
|
# ],
|
|
|
"actions": [
|
|
|
- set_root_path(),
|
|
|
- # datalad.api.run(
|
|
|
- # cmd="python3 src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
- # "fetch_submissions_bur.py",
|
|
|
- # dataset=root_path,
|
|
|
- # message="Fetch BUR submissions",
|
|
|
- # outputs="downloaded_data/UNFCCC/submissions-bur.csv",
|
|
|
- # dry_run=None,
|
|
|
- # explicit=True,
|
|
|
- # )
|
|
|
+ (fetch_bur,),
|
|
|
],
|
|
|
"verbosity": 2,
|
|
|
"setup": ["in_venv"],
|
|
|
@@ -137,12 +131,9 @@ def task_update_bur():
|
|
|
|
|
|
def task_download_bur():
|
|
|
"""Download BUR submissions"""
|
|
|
- return {
|
|
|
- #'file_dep': ['downloaded_data/UNFCCC/submissions-bur.csv'],
|
|
|
- # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
- # before download
|
|
|
- "actions": [
|
|
|
- set_root_path(),
|
|
|
+
|
|
|
+ def download_bur():
|
|
|
+ (
|
|
|
datalad.api.run(
|
|
|
cmd="python3 src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
"download_nonannexI.py --category=BUR",
|
|
|
@@ -152,448 +143,458 @@ def task_download_bur():
|
|
|
dry_run=None,
|
|
|
explicit=False,
|
|
|
),
|
|
|
- map_folders("downloaded_data/UNFCCC"),
|
|
|
- ],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
+ )
|
|
|
|
|
|
-def task_update_nc():
|
|
|
- """Update list of NC submissions"""
|
|
|
return {
|
|
|
- "targets": ["downloaded_data/UNFCCC/submissions-nc.csv"],
|
|
|
- "actions": [
|
|
|
- set_root_path(),
|
|
|
- datalad.api.run(
|
|
|
- cmd="python3 src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
- "fetch_submissions_nc.py",
|
|
|
- dataset=root_path,
|
|
|
- message="Fetch NC submissions",
|
|
|
- outputs="downloaded_data/UNFCCC/submissions-nc.csv",
|
|
|
- dry_run=None,
|
|
|
- explicit=True,
|
|
|
- ),
|
|
|
- ],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def task_download_nc():
|
|
|
- """Download NC submissions"""
|
|
|
- return {
|
|
|
- #'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
- # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
- # before download
|
|
|
- "actions": [
|
|
|
- set_root_path(),
|
|
|
- datalad.api.run(
|
|
|
- cmd="python3 src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
- "download_nonannexI.py --category=NC",
|
|
|
- dataset=root_path,
|
|
|
- message="Download NC submissions",
|
|
|
- inputs="downloaded_data/UNFCCC/submissions-nc.csv",
|
|
|
- dry_run=None,
|
|
|
- explicit=False,
|
|
|
- ),
|
|
|
- map_folders("downloaded_data/UNFCCC"),
|
|
|
- ],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-# annexI data: one update call for all data types (as they are on one page)
|
|
|
-# but for each year separately.
|
|
|
-# downloading is per year and
|
|
|
-update_aI_config = {
|
|
|
- "year": get_var("year", None),
|
|
|
- "category": get_var("category", None),
|
|
|
-}
|
|
|
-
|
|
|
-
|
|
|
-def task_update_annexi():
|
|
|
- """Update list of AnnexI submissions"""
|
|
|
- return {
|
|
|
- "targets": [
|
|
|
- f"downloaded_data/UNFCCC/submissions-annexI_{update_aI_config['year']}.csv"
|
|
|
- ],
|
|
|
- "actions": [
|
|
|
- set_root_path(),
|
|
|
- datalad.api.run(
|
|
|
- cmd="python src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
- "fetch_submissions_annexI.py "
|
|
|
- f"--year={update_aI_config['year']}",
|
|
|
- dataset=root_path,
|
|
|
- message=f"Fetch AnnexI submissions for {update_aI_config['year']}",
|
|
|
- outputs=f"downloaded_data/UNFCCC/submissions-annexI_"
|
|
|
- f"{update_aI_config['year']}.csv",
|
|
|
- dry_run=None,
|
|
|
- explicit=True,
|
|
|
- ),
|
|
|
- ],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def task_download_annexi():
|
|
|
- """Download AnnexI submissions"""
|
|
|
- return {
|
|
|
- #'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
- # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
- # before download
|
|
|
- "actions": [
|
|
|
- set_root_path(),
|
|
|
- datalad.api.run(
|
|
|
- cmd="python src/unfccc_ghg_data/unfccc_downloader/download_annexI.py "
|
|
|
- f"--category={update_aI_config['category']} "
|
|
|
- f"--year={update_aI_config['year']}",
|
|
|
- dataset=root_path,
|
|
|
- message=f"Download AnnexI submissions for "
|
|
|
- f"{update_aI_config['category']}"
|
|
|
- f"{update_aI_config['year']}",
|
|
|
- inputs=f"downloaded_data/UNFCCC/submissions-annexI_"
|
|
|
- f"{update_aI_config['year']}.csv",
|
|
|
- dry_run=None,
|
|
|
- explicit=False,
|
|
|
- ),
|
|
|
- map_folders("downloaded_data/UNFCCC"),
|
|
|
- ],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-# annexI data: one update call for all data types (as they are on one page)
|
|
|
-# but for each year separately.
|
|
|
-# downloading is per year and
|
|
|
-update_btr_config = {
|
|
|
- "round": get_var("round", None),
|
|
|
-}
|
|
|
-
|
|
|
-
|
|
|
-def task_update_btr():
|
|
|
- """Update list of BTR submissions"""
|
|
|
- return {
|
|
|
- "targets": [
|
|
|
- f"downloaded_data/UNFCCC/submissions-BTR{update_btr_config['round']}.csv"
|
|
|
- ],
|
|
|
- "actions": [
|
|
|
- set_root_path(),
|
|
|
- datalad.api.run(
|
|
|
- cmd="python src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
- "fetch_submissions_btr.py "
|
|
|
- f"--round={update_btr_config['round']}",
|
|
|
- dataset=root_path,
|
|
|
- message=f"Fetch Biannial Transparency Report submissions for "
|
|
|
- f"BTR{update_btr_config['round']}",
|
|
|
- outputs=f"downloaded_data/UNFCCC/submissions-BTR"
|
|
|
- f"{update_btr_config['round']}.csv",
|
|
|
- dry_run=None,
|
|
|
- explicit=True,
|
|
|
- ),
|
|
|
- ],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def task_download_btr():
|
|
|
- """Download BTR submissions"""
|
|
|
- return {
|
|
|
- #'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
+ #'file_dep': ['downloaded_data/UNFCCC/submissions-bur.csv'],
|
|
|
# deactivate file_dep fow now as it will always run fetch submissions
|
|
|
# before download
|
|
|
"actions": [
|
|
|
- set_root_path(),
|
|
|
- datalad.api.run(
|
|
|
- cmd="src/unfccc_ghg_data/unfccc_downloader/download_btr.py "
|
|
|
- f"--round={update_btr_config['round']}",
|
|
|
- dataset=root_path,
|
|
|
- message="Download BTR submissions for "
|
|
|
- f"BTR{update_btr_config['round']}",
|
|
|
- inputs=f"downloaded_data/UNFCCC/submissions-BTR"
|
|
|
- f"{update_btr_config['round']}.csv",
|
|
|
- dry_run=None,
|
|
|
- explicit=False,
|
|
|
- ),
|
|
|
- map_folders("downloaded_data/UNFCCC"),
|
|
|
- ],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def task_download_ndc():
|
|
|
- """Download NDC submissions"""
|
|
|
- return {
|
|
|
- "actions": [
|
|
|
- set_root_path(),
|
|
|
- datalad.api.run(
|
|
|
- cmd="src/unfccc_ghg_data/unfccc_downloader/download_ndc.py",
|
|
|
- dataset=root_path,
|
|
|
- message="Download NDC submissions",
|
|
|
- inputs=None,
|
|
|
- dry_run=None,
|
|
|
- explicit=False,
|
|
|
- ),
|
|
|
- map_folders("downloaded_data/UNFCCC"),
|
|
|
+ (download_bur,),
|
|
|
+ (map_folders, ["downloaded_data/UNFCCC"]),
|
|
|
],
|
|
|
"verbosity": 2,
|
|
|
"setup": ["in_venv"],
|
|
|
}
|
|
|
|
|
|
|
|
|
-# read UNFCCC submissions.
|
|
|
-# datalad run is called from within the read_UNFCCC_submission.py script
|
|
|
-read_config = {
|
|
|
- "country": get_var("country", None),
|
|
|
- "submission": get_var("submission", None),
|
|
|
-}
|
|
|
-
|
|
|
-
|
|
|
-# TODO: make individual task for non-UNFCCC submissions
|
|
|
-def task_read_unfccc_submission():
|
|
|
- """Read submission for a country (if code exists) (not for CRF)"""
|
|
|
- return {
|
|
|
- # "actions": [
|
|
|
- # f"python src/unfccc_ghg_data/unfccc_reader/read_UNFCCC_submission.py "
|
|
|
- # f"--country={read_config['country']} --submission={read_config['submission']}",
|
|
|
- # "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- # "--folder=extracted_data/UNFCCC",
|
|
|
- # ],
|
|
|
- "actions": [
|
|
|
- set_root_path(),
|
|
|
- datalad.api.run(
|
|
|
- cmd="src/unfccc_ghg_data/unfccc_reader/read_UNFCCC_submission.py"
|
|
|
- f"--country={read_config['country']} "
|
|
|
- f"--submission={read_config['submission']}",
|
|
|
- dataset=root_path,
|
|
|
- message=f"Read {read_config['submission']} for country "
|
|
|
- f"{read_config['country']}",
|
|
|
- dry_run=None,
|
|
|
- explicit=False,
|
|
|
- ),
|
|
|
- map_folders("extracted_data/UNFCCC"),
|
|
|
- ],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-# read UNFCCC submissions.
|
|
|
-# datalad run is called from within the read_UNFCCC_submission.py script
|
|
|
-read_config_crf = {
|
|
|
- "country": get_var("country", None),
|
|
|
- "submission_year": get_var("submission_year", None),
|
|
|
- "submission_date": get_var("submission_date", None),
|
|
|
- "re_read": get_var("re_read", False),
|
|
|
- "countries": get_var("countries", None),
|
|
|
- "data_year": get_var("data_year", None),
|
|
|
- "totest": get_var("totest", None),
|
|
|
-}
|
|
|
-
|
|
|
-
|
|
|
-def task_read_unfccc_crf_submission():
|
|
|
- """Read CRF submission for a country"""
|
|
|
- actions = [
|
|
|
- f"python src/unfccc_ghg_data/unfccc_crf_reader"
|
|
|
- f"/read_unfccc_crf_submission_datalad.py "
|
|
|
- f"--country={read_config_crf['country']} "
|
|
|
- f"--submission_year={read_config_crf['submission_year']} "
|
|
|
- f"--submission_date={read_config_crf['submission_date']} ",
|
|
|
- "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- "--folder=extracted_data/UNFCCC",
|
|
|
- ]
|
|
|
- if read_config_crf["re_read"] == "True":
|
|
|
- actions[0] = actions[0] + " --re_read"
|
|
|
- return {
|
|
|
- "actions": actions,
|
|
|
- "task_dep": ["set_env"],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def task_read_new_unfccc_crf_for_year():
|
|
|
- """
|
|
|
- Read CRF submission for all countries for given submission year.
|
|
|
-
|
|
|
- By default only reads data not present yet. Only reads the latest updated
|
|
|
- submission for each country.
|
|
|
- """
|
|
|
- actions = [
|
|
|
- f"python src/unfccc_ghg_data/unfccc_crf_reader"
|
|
|
- f"/read_new_unfccc_crf_for_year_datalad.py "
|
|
|
- f"--submission_year={read_config_crf['submission_year']} ",
|
|
|
- "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- "--folder=extracted_data/UNFCCC",
|
|
|
- ]
|
|
|
- # specifying countries is currently disabled duo to problems with command line
|
|
|
- # list arguments
|
|
|
- # if read_config_crf["countries"] is not None:
|
|
|
- # actions[0] = actions[0] + f"--countries={read_config_crf['countries']} "
|
|
|
- if read_config_crf["re_read"] == "True":
|
|
|
- actions[0] = actions[0] + " --re_read"
|
|
|
- return {
|
|
|
- #'basename': "Read_CRF_year",
|
|
|
- "actions": actions,
|
|
|
- "task_dep": ["set_env"],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def task_test_read_unfccc_crf_for_year():
|
|
|
- """
|
|
|
- Test CRF reading.
|
|
|
-
|
|
|
- Test CRF with a single year only for speed and logging to extend specifications
|
|
|
- if necessary.
|
|
|
- """
|
|
|
- actions = [
|
|
|
- f"python "
|
|
|
- f"src/unfccc_ghg_data/unfccc_crf_reader"
|
|
|
- f"/test_read_unfccc_crf_for_year.py "
|
|
|
- f"--submission_year={read_config_crf['submission_year']} "
|
|
|
- f"--country={read_config_crf['country']} "
|
|
|
- ]
|
|
|
- if read_config_crf["totest"] == "True":
|
|
|
- actions[0] = actions[0] + " --totest"
|
|
|
-
|
|
|
- if read_config_crf["data_year"] is not None:
|
|
|
- actions[0] = actions[0] + f"--data_year={read_config_crf['data_year']} "
|
|
|
- return {
|
|
|
- #'basename': "Read_CRF_year",
|
|
|
- "actions": actions,
|
|
|
- "task_dep": ["set_env"],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def task_compile_raw_unfccc_crf_for_year():
|
|
|
- """
|
|
|
- Collect all latest CRF submissions for a given year
|
|
|
-
|
|
|
- Reads the latest data fromt he extracted data folder for each country.
|
|
|
- Notifies the user if new data are available in the downloaded_data folder
|
|
|
- which have not yet been read.
|
|
|
-
|
|
|
- Data are saved in the datasets/UNFCCC/CRFYYYY folder.
|
|
|
- """
|
|
|
- actions = [
|
|
|
- f"python "
|
|
|
- f"src/unfccc_ghg_data/unfccc_crf_reader/crf_raw_for_year.py "
|
|
|
- f"--submission_year={read_config_crf['submission_year']} "
|
|
|
- ]
|
|
|
- return {
|
|
|
- "actions": actions,
|
|
|
- "task_dep": ["set_env"],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-# tasks for DI reader
|
|
|
-# datalad run is called from within the read_unfccc_di_for_country.py script
|
|
|
-read_config_di = {
|
|
|
- "country": get_var("country", None),
|
|
|
- "date": get_var("date", None),
|
|
|
- "annexI": get_var("annexI", False),
|
|
|
- # "countries": get_var('countries', None),
|
|
|
-}
|
|
|
-
|
|
|
-
|
|
|
-def task_read_unfccc_di_for_country():
|
|
|
- """Read DI data for a country"""
|
|
|
- actions = [
|
|
|
- f"python "
|
|
|
- f"src/unfccc_ghg_data/unfccc_di_reader/read_unfccc_di_for_country_datalad.py "
|
|
|
- f"--country={read_config_di['country']}",
|
|
|
- "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- "--folder=extracted_data/UNFCCC",
|
|
|
- ]
|
|
|
- return {
|
|
|
- "actions": actions,
|
|
|
- "task_dep": ["set_env"],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def task_process_unfccc_di_for_country():
|
|
|
- """Process DI data for a country"""
|
|
|
- actions = [
|
|
|
- f"python "
|
|
|
- f"src/unfccc_ghg_data/unfccc_di_reader/process_unfccc_di_for_country_datalad"
|
|
|
- f".py "
|
|
|
- f"--country={read_config_di['country']} --date={read_config_di['date']}",
|
|
|
- "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- "--folder=extracted_data/UNFCCC",
|
|
|
- ]
|
|
|
- return {
|
|
|
- "actions": actions,
|
|
|
- "task_dep": ["set_env"],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def task_read_unfccc_di_for_country_group():
|
|
|
- """Read DI data for a country group"""
|
|
|
- actions = [
|
|
|
- "python "
|
|
|
- "src/unfccc_ghg_data/unfccc_di_reader/read_unfccc_di_for_country_group_datalad"
|
|
|
- ".py",
|
|
|
- "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
- "--folder=extracted_data/UNFCCC",
|
|
|
- ]
|
|
|
- if read_config_di["annexI"] == "True":
|
|
|
- actions[0] = actions[0] + " --annexI"
|
|
|
-
|
|
|
- return {
|
|
|
- "actions": actions,
|
|
|
- "task_dep": ["set_env"],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def task_process_unfccc_di_for_country_group():
|
|
|
- """Process DI data for a country group"""
|
|
|
- actions = [
|
|
|
- "python "
|
|
|
- "src/unfccc_ghg_data/unfccc_di_reader"
|
|
|
- "/process_unfccc_di_for_country_group_datalad"
|
|
|
- ".py",
|
|
|
- ]
|
|
|
- if read_config_di["annexI"] == "True":
|
|
|
- actions[0] = actions[0] + " --annexI"
|
|
|
- if read_config_di["date"] is not None:
|
|
|
- actions[0] = actions[0] + f" --date={read_config_di['date']}"
|
|
|
-
|
|
|
- return {
|
|
|
- "actions": actions,
|
|
|
- "task_dep": ["set_env"],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-# general tasks
|
|
|
-def task_country_info():
|
|
|
- """
|
|
|
- Print information on submissions and datasets available for given country
|
|
|
- """
|
|
|
- return {
|
|
|
- "actions": [
|
|
|
- f"python src/unfccc_ghg_data/helper/country_info.py "
|
|
|
- f"--country={read_config['country']}"
|
|
|
- ],
|
|
|
- "task_dep": ["set_env"],
|
|
|
- "verbosity": 2,
|
|
|
- "setup": ["in_venv"],
|
|
|
- }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_update_nc():
|
|
|
+# """Update list of NC submissions"""
|
|
|
+# return {
|
|
|
+# "targets": ["downloaded_data/UNFCCC/submissions-nc.csv"],
|
|
|
+# "actions": [
|
|
|
+# set_root_path(),
|
|
|
+# datalad.api.run(
|
|
|
+# cmd="python3 src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
+# "fetch_submissions_nc.py",
|
|
|
+# dataset=root_path,
|
|
|
+# message="Fetch NC submissions",
|
|
|
+# outputs="downloaded_data/UNFCCC/submissions-nc.csv",
|
|
|
+# dry_run=None,
|
|
|
+# explicit=True,
|
|
|
+# ),
|
|
|
+# ],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_download_nc():
|
|
|
+# """Download NC submissions"""
|
|
|
+# return {
|
|
|
+# #'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
+# # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
+# # before download
|
|
|
+# "actions": [
|
|
|
+# set_root_path(),
|
|
|
+# datalad.api.run(
|
|
|
+# cmd="python3 src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
+# "download_nonannexI.py --category=NC",
|
|
|
+# dataset=root_path,
|
|
|
+# message="Download NC submissions",
|
|
|
+# inputs="downloaded_data/UNFCCC/submissions-nc.csv",
|
|
|
+# dry_run=None,
|
|
|
+# explicit=False,
|
|
|
+# ),
|
|
|
+# map_folders("downloaded_data/UNFCCC"),
|
|
|
+# ],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# # annexI data: one update call for all data types (as they are on one page)
|
|
|
+# # but for each year separately.
|
|
|
+# # downloading is per year and
|
|
|
+# update_aI_config = {
|
|
|
+# "year": get_var("year", None),
|
|
|
+# "category": get_var("category", None),
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_update_annexi():
|
|
|
+# """Update list of AnnexI submissions"""
|
|
|
+# return {
|
|
|
+# "targets": [
|
|
|
+# f"downloaded_data/UNFCCC/submissions-annexI_{update_aI_config['year']}.csv"
|
|
|
+# ],
|
|
|
+# "actions": [
|
|
|
+# set_root_path(),
|
|
|
+# datalad.api.run(
|
|
|
+# cmd="python src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
+# "fetch_submissions_annexI.py "
|
|
|
+# f"--year={update_aI_config['year']}",
|
|
|
+# dataset=root_path,
|
|
|
+# message=f"Fetch AnnexI submissions for {update_aI_config['year']}",
|
|
|
+# outputs=f"downloaded_data/UNFCCC/submissions-annexI_"
|
|
|
+# f"{update_aI_config['year']}.csv",
|
|
|
+# dry_run=None,
|
|
|
+# explicit=True,
|
|
|
+# ),
|
|
|
+# ],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_download_annexi():
|
|
|
+# """Download AnnexI submissions"""
|
|
|
+# return {
|
|
|
+# #'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
+# # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
+# # before download
|
|
|
+# "actions": [
|
|
|
+# set_root_path(),
|
|
|
+# datalad.api.run(
|
|
|
+# cmd="python src/unfccc_ghg_data/unfccc_downloader/download_annexI.py "
|
|
|
+# f"--category={update_aI_config['category']} "
|
|
|
+# f"--year={update_aI_config['year']}",
|
|
|
+# dataset=root_path,
|
|
|
+# message=f"Download AnnexI submissions for "
|
|
|
+# f"{update_aI_config['category']}"
|
|
|
+# f"{update_aI_config['year']}",
|
|
|
+# inputs=f"downloaded_data/UNFCCC/submissions-annexI_"
|
|
|
+# f"{update_aI_config['year']}.csv",
|
|
|
+# dry_run=None,
|
|
|
+# explicit=False,
|
|
|
+# ),
|
|
|
+# map_folders("downloaded_data/UNFCCC"),
|
|
|
+# ],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# # annexI data: one update call for all data types (as they are on one page)
|
|
|
+# # but for each year separately.
|
|
|
+# # downloading is per year and
|
|
|
+# update_btr_config = {
|
|
|
+# "round": get_var("round", None),
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_update_btr():
|
|
|
+# """Update list of BTR submissions"""
|
|
|
+# return {
|
|
|
+# "targets": [
|
|
|
+# f"downloaded_data/UNFCCC/submissions-BTR{update_btr_config['round']}.csv"
|
|
|
+# ],
|
|
|
+# "actions": [
|
|
|
+# set_root_path(),
|
|
|
+# datalad.api.run(
|
|
|
+# cmd="python src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
+# "fetch_submissions_btr.py "
|
|
|
+# f"--round={update_btr_config['round']}",
|
|
|
+# dataset=root_path,
|
|
|
+# message=f"Fetch Biannial Transparency Report submissions for "
|
|
|
+# f"BTR{update_btr_config['round']}",
|
|
|
+# outputs=f"downloaded_data/UNFCCC/submissions-BTR"
|
|
|
+# f"{update_btr_config['round']}.csv",
|
|
|
+# dry_run=None,
|
|
|
+# explicit=True,
|
|
|
+# ),
|
|
|
+# ],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_download_btr():
|
|
|
+# """Download BTR submissions"""
|
|
|
+# return {
|
|
|
+# #'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
+# # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
+# # before download
|
|
|
+# "actions": [
|
|
|
+# set_root_path(),
|
|
|
+# datalad.api.run(
|
|
|
+# cmd="src/unfccc_ghg_data/unfccc_downloader/download_btr.py "
|
|
|
+# f"--round={update_btr_config['round']}",
|
|
|
+# dataset=root_path,
|
|
|
+# message="Download BTR submissions for "
|
|
|
+# f"BTR{update_btr_config['round']}",
|
|
|
+# inputs=f"downloaded_data/UNFCCC/submissions-BTR"
|
|
|
+# f"{update_btr_config['round']}.csv",
|
|
|
+# dry_run=None,
|
|
|
+# explicit=False,
|
|
|
+# ),
|
|
|
+# map_folders("downloaded_data/UNFCCC"),
|
|
|
+# ],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_download_ndc():
|
|
|
+# """Download NDC submissions"""
|
|
|
+# return {
|
|
|
+# "actions": [
|
|
|
+# set_root_path(),
|
|
|
+# datalad.api.run(
|
|
|
+# cmd="src/unfccc_ghg_data/unfccc_downloader/download_ndc.py",
|
|
|
+# dataset=root_path,
|
|
|
+# message="Download NDC submissions",
|
|
|
+# inputs=None,
|
|
|
+# dry_run=None,
|
|
|
+# explicit=False,
|
|
|
+# ),
|
|
|
+# map_folders("downloaded_data/UNFCCC"),
|
|
|
+# ],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# # read UNFCCC submissions.
|
|
|
+# # datalad run is called from within the read_UNFCCC_submission.py script
|
|
|
+# read_config = {
|
|
|
+# "country": get_var("country", None),
|
|
|
+# "submission": get_var("submission", None),
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# # TODO: make individual task for non-UNFCCC submissions
|
|
|
+# def task_read_unfccc_submission():
|
|
|
+# """Read submission for a country (if code exists) (not for CRF)"""
|
|
|
+# return {
|
|
|
+# # "actions": [
|
|
|
+# # f"python src/unfccc_ghg_data/unfccc_reader/read_UNFCCC_submission.py "
|
|
|
+# # f"--country={read_config['country']} --submission={read_config['submission']}",
|
|
|
+# # "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+# # "--folder=extracted_data/UNFCCC",
|
|
|
+# # ],
|
|
|
+# "actions": [
|
|
|
+# set_root_path(),
|
|
|
+# datalad.api.run(
|
|
|
+# cmd="src/unfccc_ghg_data/unfccc_reader/read_UNFCCC_submission.py"
|
|
|
+# f"--country={read_config['country']} "
|
|
|
+# f"--submission={read_config['submission']}",
|
|
|
+# dataset=root_path,
|
|
|
+# message=f"Read {read_config['submission']} for country "
|
|
|
+# f"{read_config['country']}",
|
|
|
+# dry_run=None,
|
|
|
+# explicit=False,
|
|
|
+# ),
|
|
|
+# map_folders("extracted_data/UNFCCC"),
|
|
|
+# ],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# # read UNFCCC submissions.
|
|
|
+# # datalad run is called from within the read_UNFCCC_submission.py script
|
|
|
+# read_config_crf = {
|
|
|
+# "country": get_var("country", None),
|
|
|
+# "submission_year": get_var("submission_year", None),
|
|
|
+# "submission_date": get_var("submission_date", None),
|
|
|
+# "re_read": get_var("re_read", False),
|
|
|
+# "countries": get_var("countries", None),
|
|
|
+# "data_year": get_var("data_year", None),
|
|
|
+# "totest": get_var("totest", None),
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_read_unfccc_crf_submission():
|
|
|
+# """Read CRF submission for a country"""
|
|
|
+# actions = [
|
|
|
+# f"python src/unfccc_ghg_data/unfccc_crf_reader"
|
|
|
+# f"/read_unfccc_crf_submission_datalad.py "
|
|
|
+# f"--country={read_config_crf['country']} "
|
|
|
+# f"--submission_year={read_config_crf['submission_year']} "
|
|
|
+# f"--submission_date={read_config_crf['submission_date']} ",
|
|
|
+# "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+# "--folder=extracted_data/UNFCCC",
|
|
|
+# ]
|
|
|
+# if read_config_crf["re_read"] == "True":
|
|
|
+# actions[0] = actions[0] + " --re_read"
|
|
|
+# return {
|
|
|
+# "actions": actions,
|
|
|
+# "task_dep": ["set_env"],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_read_new_unfccc_crf_for_year():
|
|
|
+# """
|
|
|
+# Read CRF submission for all countries for given submission year.
|
|
|
+#
|
|
|
+# By default only reads data not present yet. Only reads the latest updated
|
|
|
+# submission for each country.
|
|
|
+# """
|
|
|
+# actions = [
|
|
|
+# f"python src/unfccc_ghg_data/unfccc_crf_reader"
|
|
|
+# f"/read_new_unfccc_crf_for_year_datalad.py "
|
|
|
+# f"--submission_year={read_config_crf['submission_year']} ",
|
|
|
+# "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+# "--folder=extracted_data/UNFCCC",
|
|
|
+# ]
|
|
|
+# # specifying countries is currently disabled duo to problems with command line
|
|
|
+# # list arguments
|
|
|
+# # if read_config_crf["countries"] is not None:
|
|
|
+# # actions[0] = actions[0] + f"--countries={read_config_crf['countries']} "
|
|
|
+# if read_config_crf["re_read"] == "True":
|
|
|
+# actions[0] = actions[0] + " --re_read"
|
|
|
+# return {
|
|
|
+# #'basename': "Read_CRF_year",
|
|
|
+# "actions": actions,
|
|
|
+# "task_dep": ["set_env"],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_test_read_unfccc_crf_for_year():
|
|
|
+# """
|
|
|
+# Test CRF reading.
|
|
|
+#
|
|
|
+# Test CRF with a single year only for speed and logging to extend specifications
|
|
|
+# if necessary.
|
|
|
+# """
|
|
|
+# actions = [
|
|
|
+# f"python "
|
|
|
+# f"src/unfccc_ghg_data/unfccc_crf_reader"
|
|
|
+# f"/test_read_unfccc_crf_for_year.py "
|
|
|
+# f"--submission_year={read_config_crf['submission_year']} "
|
|
|
+# f"--country={read_config_crf['country']} "
|
|
|
+# ]
|
|
|
+# if read_config_crf["totest"] == "True":
|
|
|
+# actions[0] = actions[0] + " --totest"
|
|
|
+#
|
|
|
+# if read_config_crf["data_year"] is not None:
|
|
|
+# actions[0] = actions[0] + f"--data_year={read_config_crf['data_year']} "
|
|
|
+# return {
|
|
|
+# #'basename': "Read_CRF_year",
|
|
|
+# "actions": actions,
|
|
|
+# "task_dep": ["set_env"],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_compile_raw_unfccc_crf_for_year():
|
|
|
+# """
|
|
|
+# Collect all latest CRF submissions for a given year
|
|
|
+#
|
|
|
+# Reads the latest data fromt he extracted data folder for each country.
|
|
|
+# Notifies the user if new data are available in the downloaded_data folder
|
|
|
+# which have not yet been read.
|
|
|
+#
|
|
|
+# Data are saved in the datasets/UNFCCC/CRFYYYY folder.
|
|
|
+# """
|
|
|
+# actions = [
|
|
|
+# f"python "
|
|
|
+# f"src/unfccc_ghg_data/unfccc_crf_reader/crf_raw_for_year.py "
|
|
|
+# f"--submission_year={read_config_crf['submission_year']} "
|
|
|
+# ]
|
|
|
+# return {
|
|
|
+# "actions": actions,
|
|
|
+# "task_dep": ["set_env"],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# # tasks for DI reader
|
|
|
+# # datalad run is called from within the read_unfccc_di_for_country.py script
|
|
|
+# read_config_di = {
|
|
|
+# "country": get_var("country", None),
|
|
|
+# "date": get_var("date", None),
|
|
|
+# "annexI": get_var("annexI", False),
|
|
|
+# # "countries": get_var('countries', None),
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_read_unfccc_di_for_country():
|
|
|
+# """Read DI data for a country"""
|
|
|
+# actions = [
|
|
|
+# f"python "
|
|
|
+# f"src/unfccc_ghg_data/unfccc_di_reader/read_unfccc_di_for_country_datalad.py "
|
|
|
+# f"--country={read_config_di['country']}",
|
|
|
+# "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+# "--folder=extracted_data/UNFCCC",
|
|
|
+# ]
|
|
|
+# return {
|
|
|
+# "actions": actions,
|
|
|
+# "task_dep": ["set_env"],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_process_unfccc_di_for_country():
|
|
|
+# """Process DI data for a country"""
|
|
|
+# actions = [
|
|
|
+# f"python "
|
|
|
+# f"src/unfccc_ghg_data/unfccc_di_reader/process_unfccc_di_for_country_datalad"
|
|
|
+# f".py "
|
|
|
+# f"--country={read_config_di['country']} --date={read_config_di['date']}",
|
|
|
+# "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+# "--folder=extracted_data/UNFCCC",
|
|
|
+# ]
|
|
|
+# return {
|
|
|
+# "actions": actions,
|
|
|
+# "task_dep": ["set_env"],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_read_unfccc_di_for_country_group():
|
|
|
+# """Read DI data for a country group"""
|
|
|
+# actions = [
|
|
|
+# "python "
|
|
|
+# "src/unfccc_ghg_data/unfccc_di_reader/read_unfccc_di_for_country_group_datalad"
|
|
|
+# ".py",
|
|
|
+# "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
+# "--folder=extracted_data/UNFCCC",
|
|
|
+# ]
|
|
|
+# if read_config_di["annexI"] == "True":
|
|
|
+# actions[0] = actions[0] + " --annexI"
|
|
|
+#
|
|
|
+# return {
|
|
|
+# "actions": actions,
|
|
|
+# "task_dep": ["set_env"],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# def task_process_unfccc_di_for_country_group():
|
|
|
+# """Process DI data for a country group"""
|
|
|
+# actions = [
|
|
|
+# "python "
|
|
|
+# "src/unfccc_ghg_data/unfccc_di_reader"
|
|
|
+# "/process_unfccc_di_for_country_group_datalad"
|
|
|
+# ".py",
|
|
|
+# ]
|
|
|
+# if read_config_di["annexI"] == "True":
|
|
|
+# actions[0] = actions[0] + " --annexI"
|
|
|
+# if read_config_di["date"] is not None:
|
|
|
+# actions[0] = actions[0] + f" --date={read_config_di['date']}"
|
|
|
+#
|
|
|
+# return {
|
|
|
+# "actions": actions,
|
|
|
+# "task_dep": ["set_env"],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
|
|
+#
|
|
|
+#
|
|
|
+# # general tasks
|
|
|
+# def task_country_info():
|
|
|
+# """
|
|
|
+# Print information on submissions and datasets available for given country
|
|
|
+# """
|
|
|
+# return {
|
|
|
+# "actions": [
|
|
|
+# f"python src/unfccc_ghg_data/helper/country_info.py "
|
|
|
+# f"--country={read_config['country']}"
|
|
|
+# ],
|
|
|
+# "task_dep": ["set_env"],
|
|
|
+# "verbosity": 2,
|
|
|
+# "setup": ["in_venv"],
|
|
|
+# }
|
