work on code to run data extraction

code/UNFCCC_reader/folder_mapping.py

@@ -0,0 +1,23 @@
+# this script takes a folder as input (from doit) and
+# runs creates the mapping of subfolders to country codes
+# oir that folder
+
+import argparse
+from get_submissions_info import create_folder_mapping
+
+# Find the right function and possible input and output files and
+# read the data using datalad run.
+parser = argparse.ArgumentParser()
+parser.add_argument('--folder', help='folder name, relative to '
+                                     'repository root folder')
+args = parser.parse_args()
+folder = args.folder
+
+if 'extracted_data' in folder:
+    extracted = True
+else:
+    extracted = False
+
+# print available submissions
+print("="*10 + f" Creating folder mapping for  {folder} " + "="*10)
+create_folder_mapping(folder, extracted)

code/UNFCCC_reader/get_submissions_info.py

@@ -473,7 +473,8 @@ def create_folder_mapping(
     """
     Create a mapping from 3 letter ISO country codes to folders
     based on the subfolders of the given folder. The mapping is
-    stored in 'folder_mapping.json' in the given folder.
+    stored in 'folder_mapping.json' in the given folder. Folder
+    must be given relative to the repository root
 
     Parameters
     ----------
@@ -489,6 +490,11 @@ def create_folder_mapping(
         Nothing
 
     """
+    codepath = Path(__file__).parent
+    rootpath = codepath / ".." / ".."
+    rootpath = rootpath.resolve()
+    folder = rootpath / folder
+
     if extracted:
         folder_mapping = {}
     else:

dodo.py

@@ -1,6 +1,38 @@
 # define tasks for UNFCCC data repository
 from doit import get_var
 
+# TODO: task for folder mapping
+
+# create virtual environment
+def task_setup_venv():
+    """Create virtual environment"""
+    return {
+        'file_dep': ['code/requirements.txt'],
+        'actions': ['python3 -m venv venv',
+                    './venv/bin/pip install --upgrade pip',
+                    './venv/bin/pip install -Ur code/requirements.txt',
+                    'touch venv',],
+        'targets': ['venv'],
+        'verbosity': 2,
+    }
+
+
+# Task to create the mapping files which map folder names to ISO 3-letter country codes
+read_config_folder = {
+    "folder": get_var('folder', None),
+}
+
+def task_map_folders():
+    """
+    Create or update the folder mapping in the given folder
+    """
+    return {
+        'actions': [f"./venv/bin/python code/UNFCCC_reader/folder_mapping.py "
+                    f"--folder={read_config_folder['folder']}"],
+        'verbosity': 2,
+        'setup': ['setup_venv'],
+    }
+
 # Tasks for getting submissions and downloading them
 def task_update_bur():
     """ Update list of BUR submissions """
@@ -10,6 +42,7 @@ def task_update_bur():
                     '-o downloaded_data/UNFCCC/submissions-bur.csv '
                     './venv/bin/python code/UNFCCC_downloader/fetch_submissions_bur.py'],
         'verbosity': 2,
+        'setup': ['setup_venv'],
     }
 
 
@@ -23,6 +56,7 @@ def task_download_bur():
                     '-i downloaded_data/UNFCCC/submissions-bur.csv '
                     './venv/bin/python code/UNFCCC_downloader/download_bur.py'],
         'verbosity': 2,
+        'setup': ['setup_venv'],
     }
 
 
@@ -34,6 +68,7 @@ def task_update_nc():
                     '-o downloaded_data/UNFCCC/submissions-nc.csv '
                     './venv/bin/python code/UNFCCC_downloader/fetch_submissions_nc.py'],
         'verbosity': 2,
+        'setup': ['setup_venv'],
     }
 
 
@@ -47,6 +82,7 @@ def task_download_nc():
                     '-i downloaded_data/UNFCCC/submissions-nc.csv '
                     './venv/bin/python code/UNFCCC_downloader/download_nc.py'],
         'verbosity': 2,
+        'setup': ['setup_venv'],
     }
 
 
@@ -56,12 +92,12 @@ def task_download_ndc():
         'actions': ['datalad run -m "Download NDC submissions" '
                     './venv/bin/python code/UNFCCC_downloader/download_ndc.py'],
         'verbosity': 2,
+        'setup': ['setup_venv'],
     }
 
 
 # read UNFCCC submissions.
 # datalad run is called from within the read_UNFCCC_submission.py script
-# add parameters and pass them to script
 read_config = {
     "country": get_var('country', None),
     "submission": get_var('submission', None),
@@ -73,21 +109,7 @@ def task_read_unfccc_submission():
         'actions': [f"./venv/bin/python code/UNFCCC_reader/read_UNFCCC_submission.py "
                     f"--country={read_config['country']} --submission={read_config['submission']}"],
         'verbosity': 2,
-        'params': [{'name': 'country',
-                    'short': 'c',
-                    'long': 'country',
-                    'default': None,
-                    'help': 'name or ISO 3-letter code of the country to read data for (e.g. China)',
-                    'type': str,
-                    },
-                   {'name': 'submission',
-                    'short': 's',
-                    'long': 'submission',
-                    'default': None,
-                    'help': 'submission to read (e.g. BUR4)',
-                    'type': str,
-                    },
-                   ],
+        'setup': ['setup_venv'],
     }
 
 
@@ -98,4 +120,6 @@ def task_country_info():
         'actions': [f"./venv/bin/python code/UNFCCC_reader/country_info.py "
                     f"--country={read_config['country']}"],
         'verbosity': 2,
-    }
+        'setup': ['setup_venv'],
+    }
+