refactor a bit

.gitignore

@@ -10,4 +10,7 @@ UNFCCC_GHG_data/UNFCCC_DI_reader/test_UNFCCC_DI_reader.ipynb
 UNFCCC_GHG_data/UNFCCC_DI_reader/.ipynb_checkpoints/
 *.autosave
 #UNFCCC_GHG_data/UNFCCC_DI_reader
+build
+UNFCCC_GHG_data.egg-info
+
 

UNFCCC_GHG_data/UNFCCC_CRF_reader/UNFCCC_CRF_reader_prod.py

@@ -1,20 +1,10 @@
-#import re
-#
-#from treelib import Tree
-
-
-#import pandas as pd
 import xarray as xr
 import primap2 as pm2
-#import numpy as np
-#import pycountry
 import datalad.api
 from datetime import date
-#from pathlib import Path
 from typing import Optional, List, Dict, Union
 
 from . import crf_specifications as crf
-#import crf_specifications as crf
 
 from .UNFCCC_CRF_reader_core import read_crf_table
 from .UNFCCC_CRF_reader_core import convert_crf_table_to_pm2if

UNFCCC_GHG_data/UNFCCC_DI_reader/UNFCCC_DI_reader_config.py

@@ -1,3 +1,5 @@
+# TODO: move gas baskets to helper
+
 di_query_filters = [
     'classifications', 'measures', 'gases',
 ]
@@ -329,7 +331,100 @@ di_processing_templates = {
     # versions. So we store them here and refer to them in the processing info dict
     #AFG: not needed (newer data in BUR1), 2005, 2013 only
     #AGO: 2000, 2005 only (external key needed for some gases / sectors)
-    #TODO ALB: pre 2000 downscaling
+    'ALB': {
+        # 1990-2009, 1990-1999 need downscaling
+        'DI2023-05-24': {
+            'remove_ts': {
+                '2.A_H': { # looks wrong in 2005
+                    'category': ['2.A', '2.B', '2.C', '2.D', '2.G'],
+                    'entities': ['CO2', 'KYOTOGHG (SARGWP100)'],
+                        'time': ['2005'],
+                },
+                'Bunkers': { # Aviation and marine swappen in 2005
+                    'category': ['14423', '14424'],
+                    'entities': ['KYOTOGHG (SARGWP100)'],
+                        'time': ['2005'],
+                },
+                'Bunkers_CH4': { # 2005 looks all wrong (swap in activity data not
+                    # result?)
+                    'category': ['14423', '14424', '14637'],
+                    'entities': ['CH4', 'KYOTOGHG (SARGWP100)', 'N2O'],
+                        'time': ['2005'],
+                },
+            },
+            'downscale': { # needed for 1990, 2000, 2005-2012
+                'sectors': {
+                    '1': {
+                        'basket': '1',
+                        'basket_contents': ['1.A', '1.B'],
+                        'entities': ['CO2', 'N2O', 'CH4'],
+                        'dim': 'category (BURDI)',
+                        #'skipna_evaluation_dims': None,
+                        #'skipna': True,
+                    },
+                    '1.A': {
+                        'basket': '1.A',
+                        'basket_contents': ['1.A.1', '1.A.2', '1.A.3', '1.A.4',
+                                            '1.A.5'],
+                        'entities': ['CO2', 'N2O', 'CH4'],
+                        'dim': 'category (BURDI)',
+                        #'skipna_evaluation_dims': None,
+                        #'skipna': True,
+                    },
+                    '1.B': {
+                        'basket': '1.B',
+                        'basket_contents': ['1.B.1', '1.B.2'],
+                        'entities': ['CH4'],
+                        'dim': 'category (BURDI)',
+                        #'skipna_evaluation_dims': None,
+                        #'skipna': True,
+                    },
+                    '2': {
+                        'basket': '2',
+                        'basket_contents': ['2.A', '2.B', '2.C', '2.D', '2.E', '2.F',
+                                            '2.G'],
+                        'entities': ['CO2', 'N2O', 'CH4'],
+                        'dim': 'category (BURDI)',
+                        #'skipna_evaluation_dims': None,
+                        #'skipna': True,
+                    },
+                    '4': {
+                        'basket': '4',
+                        'basket_contents': ['4.A', '4.B', '4.C', '4.D', '4.E', '4.F',
+                                            '4.G'],
+                        'entities': ['N2O', 'CH4'],
+                        'dim': 'category (BURDI)',
+                        #'skipna_evaluation_dims': None,
+                        #'skipna': True,
+                    },
+                    '5': {
+                        'basket': '5',
+                        'basket_contents': ['5.A', '5.B', '5.C', '5.D', '5.E'],
+                        'entities': ['CO2', 'CH4'],
+                        'dim': 'category (BURDI)',
+                        #'skipna_evaluation_dims': None,
+                        #'skipna': True,
+                    },
+                    '6': {
+                        'basket': '6',
+                        'basket_contents': ['6.A', '6.B', '6.C', '6.D'],
+                        'entities': ['N2O', 'CH4'],
+                        'dim': 'category (BURDI)',
+                        #'skipna_evaluation_dims': None,
+                        #'skipna': True,
+                    },
+                    'bunkers': {
+                        'basket': '14637',
+                        'basket_contents': ['14423', '14424'],
+                        'entities': ['CO2'],
+                        'dim': 'category (BURDI)',
+                        #'skipna_evaluation_dims': None,
+                        #'skipna': True,
+                    },
+                },
+            },
+        }
+    },
     #AND: no data
     'ARE': { # 1990, 2000, 2005, 2014. some aggregation for fgases (pfcs) needed
         'DI2023-05-24': {
@@ -1512,6 +1607,10 @@ di_processing_info = {
     # the 'default' option is used if no specific option is found such that
     # processing of new versions can be done before creating a configuration for the
     # version.
+    'ALB': {
+        'default': di_processing_templates['ALB']['DI2023-05-24'],
+        'DI2023-05-24': di_processing_templates['ALB']['DI2023-05-24'],
+    },
     'ARE': {
         'default': di_processing_templates['ARE']['DI2023-05-24'],
         'DI2023-05-24': di_processing_templates['ARE']['DI2023-05-24'],

UNFCCC_GHG_data/__init__.py

@@ -1,13 +1,15 @@
 ####
 
+from . import helper
 from . import UNFCCC_reader
 from . import UNFCCC_CRF_reader
-from . import helper
-# import UNFCCC_DI_reader
-# import UNFCCC_downloader
+from . import UNFCCC_DI_reader
+from . import UNFCCC_downloader
 
 __all__ = [
+    "helper",
     "UNFCCC_reader",
     "UNFCCC_CRF_reader",
-    "helper",
+    "UNFCCC_DI_reader",
+    "UNFCCC_downloader"
 ]

UNFCCC_GHG_data/helper/functions.py

@@ -150,16 +150,17 @@ def process_data_for_country(
                     print(f"Generating {cat_to_generate} through subtraction")
                     data_agg = data_agg.expand_dims([f'category ('
                                                      f'{cat_terminology_in})'])
+
+                    data_agg = data_agg.assign_coords(
+                        coords={f'category ({cat_terminology_in})':
+                                    (f'category ({cat_terminology_in})',
+                                     [cat_to_generate])})
                     if cat_name_present:
                         cat_name = subtract_cats_current[cat_to_generate]['name']
                         data_agg = data_agg.assign_coords(
-                            coords={f'category ({cat_terminology_in})':
+                            coords={'orig_cat_name':
                                         (f'category ({cat_terminology_in})',
-                                         [cat_to_generate])})
-                    data_agg = data_agg.assign_coords(
-                        coords={'orig_cat_name':
-                                    (f'category ({cat_terminology_in})',
-                                     [cat_name])})
+                                         [cat_name])})
                     data_country = data_country.pr.merge(data_agg,
                                                          tolerance=tolerance)
                 else:
@@ -267,6 +268,7 @@ def process_data_for_country(
                     skipna=True, min_count=1)
             else:
                 try:
+                    #print(data_country.data_vars)
                     data_country[basket] = xr.full_like(data_country["CO2"],
                                                         np.nan).pr.quantify(
                         units="Gg CO2 / year")

setup.cfg

@@ -25,9 +25,11 @@ license_file = LICENSE
 packages =
     UNFCCC_GHG_data
     UNFCCC_GHG_data.UNFCCC_CRF_reader
+    UNFCCC_GHG_data.UNFCCC_CRF_reader.crf_specifications
     UNFCCC_GHG_data.UNFCCC_reader
     UNFCCC_GHG_data.UNFCCC_downloader
-    #UNFCCC_GHG_data.UNFCCC_DI_reader
+    UNFCCC_GHG_data.UNFCCC_DI_reader
+    UNFCCC_GHG_data.helper
     #UNFCCC_GHG_data.datasets
 python_requires = >=3.8
 setup_requires =