|
|
@@ -110,17 +110,12 @@ def task_update_bur():
|
|
|
dataset=root_path,
|
|
|
message="Fetch BUR submissions",
|
|
|
outputs="downloaded_data/UNFCCC/submissions-bur.csv",
|
|
|
- dry_run=True,
|
|
|
+ dry_run=None,
|
|
|
explicit=True,
|
|
|
)
|
|
|
|
|
|
return {
|
|
|
"targets": ["downloaded_data/UNFCCC/submissions-bur.csv"],
|
|
|
- # "actions": [
|
|
|
- # 'datalad run -m "Fetch BUR submissions" '
|
|
|
- # "-o downloaded_data/UNFCCC/submissions-bur.csv "
|
|
|
- # "python src/unfccc_ghg_data/unfccc_downloader/fetch_submissions_bur.py"
|
|
|
- # ],
|
|
|
"actions": [
|
|
|
(fetch_bur,),
|
|
|
],
|
|
|
@@ -158,231 +153,257 @@ def task_download_bur():
|
|
|
}
|
|
|
|
|
|
|
|
|
-#
|
|
|
-#
|
|
|
-# def task_update_nc():
|
|
|
-# """Update list of NC submissions"""
|
|
|
-# return {
|
|
|
-# "targets": ["downloaded_data/UNFCCC/submissions-nc.csv"],
|
|
|
-# "actions": [
|
|
|
-# set_root_path(),
|
|
|
-# datalad.api.run(
|
|
|
-# cmd="python3 src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
-# "fetch_submissions_nc.py",
|
|
|
-# dataset=root_path,
|
|
|
-# message="Fetch NC submissions",
|
|
|
-# outputs="downloaded_data/UNFCCC/submissions-nc.csv",
|
|
|
-# dry_run=None,
|
|
|
-# explicit=True,
|
|
|
-# ),
|
|
|
-# ],
|
|
|
-# "verbosity": 2,
|
|
|
-# "setup": ["in_venv"],
|
|
|
-# }
|
|
|
-#
|
|
|
-#
|
|
|
-# def task_download_nc():
|
|
|
-# """Download NC submissions"""
|
|
|
-# return {
|
|
|
-# #'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
-# # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
-# # before download
|
|
|
-# "actions": [
|
|
|
-# set_root_path(),
|
|
|
-# datalad.api.run(
|
|
|
-# cmd="python3 src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
-# "download_nonannexI.py --category=NC",
|
|
|
-# dataset=root_path,
|
|
|
-# message="Download NC submissions",
|
|
|
-# inputs="downloaded_data/UNFCCC/submissions-nc.csv",
|
|
|
-# dry_run=None,
|
|
|
-# explicit=False,
|
|
|
-# ),
|
|
|
-# map_folders("downloaded_data/UNFCCC"),
|
|
|
-# ],
|
|
|
-# "verbosity": 2,
|
|
|
-# "setup": ["in_venv"],
|
|
|
-# }
|
|
|
-#
|
|
|
-#
|
|
|
-# # annexI data: one update call for all data types (as they are on one page)
|
|
|
-# # but for each year separately.
|
|
|
-# # downloading is per year and
|
|
|
-# update_aI_config = {
|
|
|
-# "year": get_var("year", None),
|
|
|
-# "category": get_var("category", None),
|
|
|
-# }
|
|
|
-#
|
|
|
-#
|
|
|
-# def task_update_annexi():
|
|
|
-# """Update list of AnnexI submissions"""
|
|
|
-# return {
|
|
|
-# "targets": [
|
|
|
-# f"downloaded_data/UNFCCC/submissions-annexI_{update_aI_config['year']}.csv"
|
|
|
-# ],
|
|
|
-# "actions": [
|
|
|
-# set_root_path(),
|
|
|
-# datalad.api.run(
|
|
|
-# cmd="python src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
-# "fetch_submissions_annexI.py "
|
|
|
-# f"--year={update_aI_config['year']}",
|
|
|
-# dataset=root_path,
|
|
|
-# message=f"Fetch AnnexI submissions for {update_aI_config['year']}",
|
|
|
-# outputs=f"downloaded_data/UNFCCC/submissions-annexI_"
|
|
|
-# f"{update_aI_config['year']}.csv",
|
|
|
-# dry_run=None,
|
|
|
-# explicit=True,
|
|
|
-# ),
|
|
|
-# ],
|
|
|
-# "verbosity": 2,
|
|
|
-# "setup": ["in_venv"],
|
|
|
-# }
|
|
|
-#
|
|
|
-#
|
|
|
-# def task_download_annexi():
|
|
|
-# """Download AnnexI submissions"""
|
|
|
-# return {
|
|
|
-# #'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
-# # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
-# # before download
|
|
|
-# "actions": [
|
|
|
-# set_root_path(),
|
|
|
-# datalad.api.run(
|
|
|
-# cmd="python src/unfccc_ghg_data/unfccc_downloader/download_annexI.py "
|
|
|
-# f"--category={update_aI_config['category']} "
|
|
|
-# f"--year={update_aI_config['year']}",
|
|
|
-# dataset=root_path,
|
|
|
-# message=f"Download AnnexI submissions for "
|
|
|
-# f"{update_aI_config['category']}"
|
|
|
-# f"{update_aI_config['year']}",
|
|
|
-# inputs=f"downloaded_data/UNFCCC/submissions-annexI_"
|
|
|
-# f"{update_aI_config['year']}.csv",
|
|
|
-# dry_run=None,
|
|
|
-# explicit=False,
|
|
|
-# ),
|
|
|
-# map_folders("downloaded_data/UNFCCC"),
|
|
|
-# ],
|
|
|
-# "verbosity": 2,
|
|
|
-# "setup": ["in_venv"],
|
|
|
-# }
|
|
|
-#
|
|
|
-#
|
|
|
-# # annexI data: one update call for all data types (as they are on one page)
|
|
|
-# # but for each year separately.
|
|
|
-# # downloading is per year and
|
|
|
-# update_btr_config = {
|
|
|
-# "round": get_var("round", None),
|
|
|
-# }
|
|
|
-#
|
|
|
-#
|
|
|
-# def task_update_btr():
|
|
|
-# """Update list of BTR submissions"""
|
|
|
-# return {
|
|
|
-# "targets": [
|
|
|
-# f"downloaded_data/UNFCCC/submissions-BTR{update_btr_config['round']}.csv"
|
|
|
-# ],
|
|
|
-# "actions": [
|
|
|
-# set_root_path(),
|
|
|
-# datalad.api.run(
|
|
|
-# cmd="python src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
-# "fetch_submissions_btr.py "
|
|
|
-# f"--round={update_btr_config['round']}",
|
|
|
-# dataset=root_path,
|
|
|
-# message=f"Fetch Biannial Transparency Report submissions for "
|
|
|
-# f"BTR{update_btr_config['round']}",
|
|
|
-# outputs=f"downloaded_data/UNFCCC/submissions-BTR"
|
|
|
-# f"{update_btr_config['round']}.csv",
|
|
|
-# dry_run=None,
|
|
|
-# explicit=True,
|
|
|
-# ),
|
|
|
-# ],
|
|
|
-# "verbosity": 2,
|
|
|
-# "setup": ["in_venv"],
|
|
|
-# }
|
|
|
-#
|
|
|
-#
|
|
|
-# def task_download_btr():
|
|
|
-# """Download BTR submissions"""
|
|
|
-# return {
|
|
|
-# #'file_dep': ['downloaded_data/UNFCCC/submissions-nc.csv'],
|
|
|
-# # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
-# # before download
|
|
|
-# "actions": [
|
|
|
-# set_root_path(),
|
|
|
-# datalad.api.run(
|
|
|
-# cmd="src/unfccc_ghg_data/unfccc_downloader/download_btr.py "
|
|
|
-# f"--round={update_btr_config['round']}",
|
|
|
-# dataset=root_path,
|
|
|
-# message="Download BTR submissions for "
|
|
|
-# f"BTR{update_btr_config['round']}",
|
|
|
-# inputs=f"downloaded_data/UNFCCC/submissions-BTR"
|
|
|
-# f"{update_btr_config['round']}.csv",
|
|
|
-# dry_run=None,
|
|
|
-# explicit=False,
|
|
|
-# ),
|
|
|
-# map_folders("downloaded_data/UNFCCC"),
|
|
|
-# ],
|
|
|
-# "verbosity": 2,
|
|
|
-# "setup": ["in_venv"],
|
|
|
-# }
|
|
|
-#
|
|
|
-#
|
|
|
-# def task_download_ndc():
|
|
|
-# """Download NDC submissions"""
|
|
|
-# return {
|
|
|
-# "actions": [
|
|
|
-# set_root_path(),
|
|
|
-# datalad.api.run(
|
|
|
-# cmd="src/unfccc_ghg_data/unfccc_downloader/download_ndc.py",
|
|
|
-# dataset=root_path,
|
|
|
-# message="Download NDC submissions",
|
|
|
-# inputs=None,
|
|
|
-# dry_run=None,
|
|
|
-# explicit=False,
|
|
|
-# ),
|
|
|
-# map_folders("downloaded_data/UNFCCC"),
|
|
|
-# ],
|
|
|
-# "verbosity": 2,
|
|
|
-# "setup": ["in_venv"],
|
|
|
-# }
|
|
|
-#
|
|
|
-#
|
|
|
-# # read UNFCCC submissions.
|
|
|
-# # datalad run is called from within the read_UNFCCC_submission.py script
|
|
|
-# read_config = {
|
|
|
-# "country": get_var("country", None),
|
|
|
-# "submission": get_var("submission", None),
|
|
|
-# }
|
|
|
-#
|
|
|
-#
|
|
|
-# # TODO: make individual task for non-UNFCCC submissions
|
|
|
-# def task_read_unfccc_submission():
|
|
|
-# """Read submission for a country (if code exists) (not for CRF)"""
|
|
|
-# return {
|
|
|
-# # "actions": [
|
|
|
-# # f"python src/unfccc_ghg_data/unfccc_reader/read_UNFCCC_submission.py "
|
|
|
-# # f"--country={read_config['country']} --submission={read_config['submission']}",
|
|
|
-# # "python src/unfccc_ghg_data/helper/folder_mapping.py "
|
|
|
-# # "--folder=extracted_data/UNFCCC",
|
|
|
-# # ],
|
|
|
-# "actions": [
|
|
|
-# set_root_path(),
|
|
|
-# datalad.api.run(
|
|
|
-# cmd="src/unfccc_ghg_data/unfccc_reader/read_UNFCCC_submission.py"
|
|
|
-# f"--country={read_config['country']} "
|
|
|
-# f"--submission={read_config['submission']}",
|
|
|
-# dataset=root_path,
|
|
|
-# message=f"Read {read_config['submission']} for country "
|
|
|
-# f"{read_config['country']}",
|
|
|
-# dry_run=None,
|
|
|
-# explicit=False,
|
|
|
-# ),
|
|
|
-# map_folders("extracted_data/UNFCCC"),
|
|
|
-# ],
|
|
|
-# "verbosity": 2,
|
|
|
-# "setup": ["in_venv"],
|
|
|
-# }
|
|
|
-#
|
|
|
+def task_update_nc():
|
|
|
+ """Update list of NC submissions"""
|
|
|
+
|
|
|
+ def fetch_nc():
|
|
|
+ datalad.api.run(
|
|
|
+ cmd="python3 src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
+ "fetch_submissions_nc.py",
|
|
|
+ dataset=root_path,
|
|
|
+ message="Fetch NC submissions",
|
|
|
+ outputs="downloaded_data/UNFCCC/submissions-nc.csv",
|
|
|
+ dry_run=None,
|
|
|
+ explicit=True,
|
|
|
+ )
|
|
|
+
|
|
|
+ return {
|
|
|
+ "targets": ["downloaded_data/UNFCCC/submissions-nc.csv"],
|
|
|
+ "actions": [
|
|
|
+ (fetch_nc,),
|
|
|
+ ],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def task_download_nc():
|
|
|
+ """Download BUR submissions"""
|
|
|
+
|
|
|
+ def download_nc():
|
|
|
+ (
|
|
|
+ datalad.api.run(
|
|
|
+ cmd="python3 src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
+ "download_nonannexI.py --category=NC",
|
|
|
+ dataset=root_path,
|
|
|
+ message="Download NC submissions",
|
|
|
+ inputs="downloaded_data/UNFCCC/submissions-nc.csv",
|
|
|
+ dry_run=None,
|
|
|
+ explicit=False,
|
|
|
+ ),
|
|
|
+ )
|
|
|
+
|
|
|
+ return {
|
|
|
+ #'file_dep': ['downloaded_data/UNFCCC/submissions-bur.csv'],
|
|
|
+ # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
+ # before download
|
|
|
+ "actions": [
|
|
|
+ (download_nc,),
|
|
|
+ (map_folders, ["downloaded_data/UNFCCC"]),
|
|
|
+ ],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+# annexI data: one update call for all data types (as they are on one page)
|
|
|
+# but for each year separately.
|
|
|
+# downloading is per year and
|
|
|
+update_aI_config = {
|
|
|
+ "year": get_var("year", None),
|
|
|
+ "category": get_var("category", None),
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+def task_update_annexi():
|
|
|
+ """Update list of AnnexI submissions"""
|
|
|
+
|
|
|
+ def fetch_annexi():
|
|
|
+ (
|
|
|
+ datalad.api.run(
|
|
|
+ cmd="python src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
+ "fetch_submissions_annexI.py "
|
|
|
+ f"--year={update_aI_config['year']}",
|
|
|
+ dataset=root_path,
|
|
|
+ message=f"Fetch AnnexI submissions for {update_aI_config['year']}",
|
|
|
+ outputs=f"downloaded_data/UNFCCC/submissions-annexI_"
|
|
|
+ f"{update_aI_config['year']}.csv",
|
|
|
+ dry_run=None,
|
|
|
+ explicit=True,
|
|
|
+ ),
|
|
|
+ )
|
|
|
+
|
|
|
+ return {
|
|
|
+ "targets": [
|
|
|
+ f"downloaded_data/UNFCCC/submissions-annexI_{update_aI_config['year']}.csv"
|
|
|
+ ],
|
|
|
+ "actions": [
|
|
|
+ (fetch_annexi,),
|
|
|
+ ],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def task_download_annexi():
|
|
|
+ """Download AnnexI submissions"""
|
|
|
+
|
|
|
+ def download_annexi():
|
|
|
+ (
|
|
|
+ datalad.api.run(
|
|
|
+ cmd="python src/unfccc_ghg_data/unfccc_downloader/download_annexI.py "
|
|
|
+ f"--category={update_aI_config['category']} "
|
|
|
+ f"--year={update_aI_config['year']}",
|
|
|
+ dataset=root_path,
|
|
|
+ message=f"Download AnnexI submissions for "
|
|
|
+ f"{update_aI_config['category']}"
|
|
|
+ f"{update_aI_config['year']}",
|
|
|
+ inputs=f"downloaded_data/UNFCCC/submissions-annexI_"
|
|
|
+ f"{update_aI_config['year']}.csv",
|
|
|
+ dry_run=None,
|
|
|
+ explicit=False,
|
|
|
+ ),
|
|
|
+ )
|
|
|
+
|
|
|
+ return {
|
|
|
+ # 'file_dep': [f"downloaded_data/UNFCCC/submissions-annex1_"
|
|
|
+ # f"{update_aI_config['year']}.csv"],
|
|
|
+ # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
+ # before download
|
|
|
+ "actions": [
|
|
|
+ (download_annexi,),
|
|
|
+ (map_folders, ["downloaded_data/UNFCCC"]),
|
|
|
+ ],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+# BTR data: one update call for all data types (as they are on one page)
|
|
|
+# but for each submission round separately.
|
|
|
+# downloading is per submission round
|
|
|
+update_btr_config = {
|
|
|
+ "round": get_var("round", None),
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+def task_update_btr():
|
|
|
+ """Update list of BTR submissions"""
|
|
|
+
|
|
|
+ def fetch_btr():
|
|
|
+ (
|
|
|
+ datalad.api.run(
|
|
|
+ cmd="python src/unfccc_ghg_data/unfccc_downloader/"
|
|
|
+ "fetch_submissions_btr.py "
|
|
|
+ f"--round={update_btr_config['round']}",
|
|
|
+ dataset=root_path,
|
|
|
+ message=f"Fetch Biannial Transparency Report submissions for "
|
|
|
+ f"BTR{update_btr_config['round']}",
|
|
|
+ outputs=f"downloaded_data/UNFCCC/submissions-BTR"
|
|
|
+ f"{update_btr_config['round']}.csv",
|
|
|
+ dry_run=None,
|
|
|
+ explicit=True,
|
|
|
+ ),
|
|
|
+ )
|
|
|
+
|
|
|
+ return {
|
|
|
+ "targets": [
|
|
|
+ f"downloaded_data/UNFCCC/submissions-BTR{update_btr_config['round']}.csv"
|
|
|
+ ],
|
|
|
+ "actions": [
|
|
|
+ (fetch_btr,),
|
|
|
+ ],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def task_download_btr():
|
|
|
+ """Download BTR submissions"""
|
|
|
+
|
|
|
+ def download_btr():
|
|
|
+ (
|
|
|
+ datalad.api.run(
|
|
|
+ cmd="src/unfccc_ghg_data/unfccc_downloader/download_btr.py "
|
|
|
+ f"--round={update_btr_config['round']}",
|
|
|
+ dataset=root_path,
|
|
|
+ message="Download BTR submissions for "
|
|
|
+ f"BTR{update_btr_config['round']}",
|
|
|
+ inputs=f"downloaded_data/UNFCCC/submissions-BTR"
|
|
|
+ f"{update_btr_config['round']}.csv",
|
|
|
+ dry_run=None,
|
|
|
+ explicit=False,
|
|
|
+ ),
|
|
|
+ )
|
|
|
+
|
|
|
+ return {
|
|
|
+ # 'file_dep': [f"downloaded_data/UNFCCC/submissions-btr.csv "
|
|
|
+ # f"{update_btr_config['round']}.csv"],
|
|
|
+ # deactivate file_dep fow now as it will always run fetch submissions
|
|
|
+ # before download
|
|
|
+ "actions": [
|
|
|
+ (download_btr,),
|
|
|
+ (map_folders, ["downloaded_data/UNFCCC"]),
|
|
|
+ ],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def task_download_ndc():
|
|
|
+ """Download NDC submissions"""
|
|
|
+
|
|
|
+ def download_ndc():
|
|
|
+ (
|
|
|
+ datalad.api.run(
|
|
|
+ cmd="src/unfccc_ghg_data/unfccc_downloader/download_ndc.py",
|
|
|
+ dataset=root_path,
|
|
|
+ message="Download NDC submissions",
|
|
|
+ inputs=None,
|
|
|
+ dry_run=None,
|
|
|
+ explicit=False,
|
|
|
+ ),
|
|
|
+ )
|
|
|
+
|
|
|
+ return {
|
|
|
+ "actions": [
|
|
|
+ (download_ndc,),
|
|
|
+ (map_folders, ["downloaded_data/UNFCCC"]),
|
|
|
+ ],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+# read UNFCCC submissions.
|
|
|
+# datalad run is called from within the read_UNFCCC_submission.py script
|
|
|
+read_config = {
|
|
|
+ "country": get_var("country", None),
|
|
|
+ "submission": get_var("submission", None),
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+# TODO: make individual task for non-UNFCCC submissions
|
|
|
+def task_read_unfccc_submission():
|
|
|
+ """Read submission for a country (if code exists) (not for CRF)
|
|
|
+
|
|
|
+ Datalad is called from `read_UNFCCC_submission`, so we can just call this script
|
|
|
+ here.
|
|
|
+
|
|
|
+ TODO: check if it makes sense to convert script to function
|
|
|
+ """
|
|
|
+ return {
|
|
|
+ "actions": [
|
|
|
+ f"python src/unfccc_ghg_data/unfccc_reader/read_UNFCCC_submission.py "
|
|
|
+ f"--country={read_config['country']} "
|
|
|
+ f"--submission={read_config['submission']}",
|
|
|
+ (map_folders, ["extracted_data/UNFCCC"]),
|
|
|
+ ],
|
|
|
+ "verbosity": 2,
|
|
|
+ "setup": ["in_venv"],
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
#
|
|
|
# # read UNFCCC submissions.
|
|
|
# # datalad run is called from within the read_UNFCCC_submission.py script
|
