Add more tables to AUS CRT/CRF specification

UNFCCC_GHG_data/UNFCCC_CRF_reader/UNFCCC_CRF_reader_core.py

@@ -359,9 +359,22 @@ def read_crf_table_from_file(
     else:
         last_row_nan = False
     
-    # remove empty columns (for Australia tables start with an empty column)
-    df_raw = df_raw.dropna(how='all', axis=1)
-    
+
+    cols_to_drop = []
+    # remove empty first column (for Australia tables start with an empty column)
+    # df_raw = df_raw.dropna(how="all", axis=1)
+    if df_raw.iloc[:, 0].isna().all():
+        cols_to_drop.append(df_raw.columns.values[0])
+    # select only first table by cutting everything after a all-nan column (unless
+    # it's the first column)
+    for colIdx in range(1, len(df_raw.columns.values)):
+        if df_raw.iloc[:, colIdx].isna().all():
+            cols_to_drop = cols_to_drop + list(df_raw.columns.values[colIdx : ])
+            break
+
+    if cols_to_drop is not None:
+        df_raw = df_raw.drop(columns=cols_to_drop)
+
     #### prepare the header (2 row header, first entity, then unit)
     # We do this before removing columns and any other processing to
     # have consistent column names in the configuration and to avoid

UNFCCC_GHG_data/UNFCCC_CRF_reader/crf_specifications/CRF2023_AUS_specification.py

@@ -752,7 +752,7 @@ CRF2023_AUS = {
         "status": "tested",
         "table": {
             "firstrow": 7,
-            "lastrow": 33,
+            "lastrow": 45,
             "header": ['group', 'entity', 'unit'],
             "col_for_categories": "GREENHOUSE GAS SOURCE AND SINK CATEGORIES",
             "categories": ["category"],
@@ -1095,6 +1095,134 @@ CRF2023_AUS = {
             "class": "Total",
         },
     },  # tested
+    "Table3.A": {  # Interic fermentation
+        "status": "tested",
+        "table": {
+            "firstrow": 7,
+            "lastrow": 45,
+            "header": ['entity', 'entity', 'unit'],
+            "col_for_categories": "GREENHOUSE GAS SOURCE AND SINK CATEGORIES",
+            "categories": ["category"],
+            "cols_to_ignore": [
+                'ACTIVITY DATA AND OTHER RELATED INFORMATION Population size (1)',
+                'ACTIVITY DATA AND OTHER RELATED INFORMATION Average gross energy intake (GE)',
+                'ACTIVITY DATA AND OTHER RELATED INFORMATION Average CH4 conversion rate (Ym) (2)',
+                'IMPLIED EMISSION FACTORS CH4',
+            ],
+            "stop_cats": ["", np.nan],
+            "unit_info": unit_info["default"],
+        },
+        "sector_mapping": [
+            ['3.A.1. Cattle', ['3.A.1'], 0],
+            ['Option A:', ['\IGNORE'], 1],
+            ['3.A.1.a. Dairy cattle', ['3.A.1.Aa'], 2],
+            ['3.A.1.b. Non-dairy cattle', ['3.A.1.Ab'], 2],
+            ['Option B (country-specific): (3)', ['\IGNORE'], 1],
+            ['3.A.1.a. Other', ['\IGNORE'], 2],
+            ['Drop-down list:', ['\IGNORE'], 3],
+            ['3.A.1.a.i. Mature dairy cattle', ['\IGNORE'], 3],
+            ['3.A.1.a.ii. Other mature cattle', ['\IGNORE'], 3],
+            ['3.A.1.a.iii. Growing cattle', ['\IGNORE'], 3],
+            ['3.A.1.a.iv. Other (please specify)', ['3.A.1.C'], 3],
+            ['Dairy Cattle', ['3.A.1.C-AUS-a'], 4],
+            ['Beef Cattle - Pasture', ['3.A.1.C-AUS-b'], 4],
+            ['Beef Cattle - Feedlot', ['3.A.1.C-AUS-c'], 4],
+            ['3.A.2. Sheep', ['3.A.2'], 0],
+            ['3.A.2.a. Other (please specify)', ['\IGNORE'], 1],
+            ['Sheep', ['\IGNORE'], 2],
+            ['3.A.3. Swine', ['3.A.3'], 0],
+            ['3.A.3.a. Other (please specify)', ['\IGNORE'], 1],
+            ['Swine', ['\IGNORE'], 2],
+            ['3.A.4. Other livestock (please specify) (4)', ['3.A.4'], 0],
+            ['Drop down list:', ['\IGNORE'], 1],
+            ['3.A.4.a. Buffalo', ['3.A.4.a'], 2],
+            ['3.A.4.b. Camels', ['3.A.4.b'], 2],
+            ['3.A.4.c. Deer', ['3.A.4.c'], 2],
+            ['3.A.4.d. Goats', ['3.A.4.d'], 2],
+            ['3.A.4.e. Horses', ['3.A.4.e'], 2],
+            ['3.A.4.f. Mules and asses', ['3.A.4.f'], 2],
+            ['3.A.4.g. Poultry', ['3.A.4.g'], 2],
+            ['3.A.4.h. Other (please specify)', ['3.A.4.h'], 2],
+            ['3.A.4.h.i. Rabbit', ['3.A.4.h.i'], 3],
+            ['3.A.4.h.ii. Reindeer', ['3.A.4.h.ii'], 3],
+            ['3.A.4.h.iii. Ostrich', ['3.A.4.h.iii'], 3],
+            ['3.A.4.h.iv. Fur-bearing animals (5)', ['3.A.4.h.iv'], 3],
+            ['3.A.4.h.v. Other (please specity)', ['3.A.4.h.v'], 3],
+            ['Alpacas', ['\IGNORE'], 4],
+        ],
+        "entity_mapping": {
+            'EMISSIONS CH4': 'CH4'
+        },
+        "coords_defaults": {
+            "class": "Total",
+        },
+    },  # tested
+    "Table3.B(a)": {  # Manure management
+        "status": "tested",
+        "table": {
+            "firstrow": 6,
+            "lastrow": 45,
+            "header": ['entity', 'entity', 'entity', 'unit'],
+            "col_for_categories": "GREENHOUSE GAS SOURCE AND SINK CATEGORIES",
+            "categories": ["category"],
+            "cols_to_ignore": [
+                'ACTIVITY DATA AND OTHER RELATED INFORMATION Population size',
+                'ACTIVITY DATA AND OTHER RELATED INFORMATION Allocation by climate region (1) Cool',
+                'ACTIVITY DATA AND OTHER RELATED INFORMATION Allocation by climate region (1) Temperate',
+                'ACTIVITY DATA AND OTHER RELATED INFORMATION Allocation by climate region (1) Warm',
+                'ACTIVITY DATA AND OTHER RELATED INFORMATION Typical animal mass (average) Warm',
+                'ACTIVITY DATA AND OTHER RELATED INFORMATION VS(2) daily excretion (average) Warm',
+                'ACTIVITY DATA AND OTHER RELATED INFORMATION CH4 producing potential (Bo) (2) (average) Warm',
+                'IMPLIED EMISSION FACTORS CH4 producing potential (Bo) (2) (average) CH4',
+            ],
+            "stop_cats": ["", np.nan],
+            "unit_info": unit_info["default"],
+        },
+        "sector_mapping": [
+            ['3.B.1. Cattle', ['3.B.1'], 0],
+            ['Option A:', ['\IGNORE'], 1],
+            ['3.B.1.a. Dairy cattle', ['3.B.1.Aa'], 2],
+            ['3.B.1.b. Non-dairy cattle', ['3.B.1.Ab'], 2],
+            ['Option B (country-specific): (3)', ['\IGNORE'], 1],
+            ['3.B.1.a. Other', ['\IGNORE'], 2],
+            ['Drop down list:', ['\IGNORE'], 3],
+            ['3.B.1.a.i. Mature dairy cattle', ['\IGNORE'], 3],
+            ['3.B.1.a.ii. Other mature cattle', ['\IGNORE'], 3],
+            ['3.B.1.a.iii. Growing cattle', ['\IGNORE'], 3],
+            ['3.B.1.a.iv. Other (please specify)', ['3.B.1.C'], 3],
+            ['Dairy Cattle', ['3.B.1.C-AUS-a'], 4],
+            ['Beef Cattle - Pasture', ['3.B.1.C-AUS-b'], 4],
+            ['Beef Cattle - Feedlot', ['3.B.1.C-AUS-c'], 4],
+            ['3.B.2. Sheep', ['3.B.2'], 0],
+            ['3.B.2.a. Other (please specify)', ['\IGNORE'], 1],
+            ['Sheep', ['\IGNORE'], 2],
+            ['3.B.3. Swine', ['3.B.3'], 0],
+            ['3.B.3.a. Other (please specify)', ['\IGNORE'], 1],
+            ['Swine', ['\IGNORE'], 2],
+            ['3.B.4. Other livestock (4)', ['3.B.4'], 0],
+            ['Drop-down list:', ['\IGNORE'], 1],
+            ['3.B.4.a. Buffalo', ['3.B.4.a'], 2],
+            ['3.B.4.b. Camels', ['3.B.4.b'], 2],
+            ['3.B.4.c. Deer', ['3.B.4.c'], 2],
+            ['3.B.4.d. Goats', ['3.B.4.d'], 2],
+            ['3.B.4.e. Horses', ['3.B.4.e'], 2],
+            ['3.B.4.f. Mules and Asses', ['3.B.4.f'], 2],
+            ['3.B.4.g. Poultry', ['3.B.4.g'], 2],
+            ['3.B.4.h. Other (please specify)', ['3.B.4.h'], 2],
+            ['3.B.4.h.i. Rabbit', ['3.B.4.h.i'], 3],
+            ['3.B.4.h.ii. Reindeer', ['3.B.4.h.ii'], 3],
+            ['3.B.4.h.iii. Ostrich', ['3.B.4.h.iii'], 3],
+            ['3.B.4.h.iv. Fur-bearing animals (5)', ['3.B.4.h.iv'], 3],
+            ['3.B.4.h.v. Other (please specity)', ['3.B.4.h.v'], 3],
+            ['Alpacas', ['\IGNORE'], 4],
+        ],
+        "entity_mapping": {
+            'EMISSIONS CH4 producing potential (Bo) (2) (average) CH4': 'CH4'
+        },
+        "coords_defaults": {
+            "class": "Total",
+        },
+    },  # tested
     # TODO: tables 3.A and 3.B for livestock details as they are not contained in table3
     "Table3.C": {  # rice cultivation details
         "status": "tested",
@@ -1148,10 +1276,10 @@ CRF2023_AUS = {
                 "ACTIVITY DATA AND OTHER RELATED INFORMATION Description",
                 "ACTIVITY DATA AND OTHER RELATED INFORMATION Value",
                 "IMPLIED EMISSION FACTORS Value",
-                "Fraction (a) FracGASF",
-                "Description Fraction of synthetic fertilizer N applied to soils that "
-                "volatilises as NH3 and NOX",
-                "Value 0.11",
+                # "Fraction (a) FracGASF",
+                # "Description Fraction of synthetic fertilizer N applied to soils that "
+                # "volatilises as NH3 and NOX",
+                # "Value 0.11",
             ],
             "stop_cats": ["", np.nan],
             "unit_info": unit_info["default"],
@@ -1218,8 +1346,8 @@ CRF2023_AUS = {
     "Table3.F": {  # field burning details
         "status": "TODO",
         "table": {
-            "firstrow": 5,
-            "lastrow": 30,
+            "firstrow": 7,
+            "lastrow": 29,
             "header": ['group', 'entity', 'unit'],
             "col_for_categories": "GREENHOUSE GAS SOURCE AND SINK CATEGORIES",
             "categories": ["category"],