Common Data Fields#

This notebook will walk through how to access some of the more common data fields from the Madrigal hdf5 data files. This should be treated as a quick reference for common scientific applications and is NOT intended to be a comprehensive map of everything in these files. Refer to metadata contained within the files for descriptions of other arrays.

import h5py
import madrigalWeb.madrigalWeb
import os
import numpy as np
madrigalUrl='http://cedar.openmadrigal.org'

data = madrigalWeb.madrigalWeb.MadrigalData(madrigalUrl)

user_fullname = 'Student Example' 
user_email = 'isr.summer.school@gmail.com' 
user_affiliation= 'ISR Summer School 2024'
code = 61   # PFISR
year = 2024
month = 1
day = 8
hour1 = 7 
minute1 = 1
hour2 = 13
# list of experiments inside a time period of a day
expList = data.getExperiments(code,year,month,day,hour1,minute1,0,year,month,day,hour2,0,0)
for exp in expList:
    print(str(exp))
id: 100278619
realUrl: http://cedar.openmadrigal.org/showExperiment/?experiment_list=100278619
url: http://cedar.openmadrigal.org/madtoc/experiments4/2024/pfa/08jan24a
name: Themis36 - Auroral and convection measurements
siteid: 10
sitename: CEDAR
instcode: 61
instname: Poker Flat IS Radar
startyear: 2024
startmonth: 1
startday: 8
starthour: 7
startmin: 1
startsec: 4
endyear: 2024
endmonth: 1
endday: 8
endhour: 18
endmin: 0
endsec: 0
isLocal: True
madrigalUrl: http://cedar.openmadrigal.org/
PI: Asti Bhatt
PIEmail: asti.bhatt@sri.com
uttimestamp: 1709109883
access: 2
Madrigal version: 3.2
fileList = data.getExperimentFiles(expList[0].id)
for file0 in fileList:
    print(os.path.basename(file0.name),'\t', file0.kindat, '\t',file0.kindatdesc)
pfa20240108.001_ac_nenotr_01min.001.h5 	 1000201 	 Ne From Power - Alternating Code (E-region) - 1 min
pfa20240108.001_ac_fit_01min.001.h5 	 2000201 	 Fitted - Alternating Code (E-region) - 1 min
pfa20240108.001_ac_nenotr_03min.001.h5 	 1000203 	 Ne From Power - Alternating Code (E-region) - 3 min
pfa20240108.001_ac_fit_03min.001.h5 	 2000203 	 Fitted - Alternating Code (E-region) - 3 min
pfa20240108.001_ac_nenotr_05min.001.h5 	 1000205 	 Ne From Power - Alternating Code (E-region) - 5 min
pfa20240108.001_ac_fit_05min.001.h5 	 2000205 	 Fitted - Alternating Code (E-region) - 5 min
pfa20240108.001_ac_nenotr_10min.001.h5 	 1000210 	 Ne From Power - Alternating Code (E-region) - 10 min
pfa20240108.001_ac_fit_10min.001.h5 	 2000210 	 Fitted - Alternating Code (E-region) - 10 min
pfa20240108.001_ac_nenotr_15min.001.h5 	 1000215 	 Ne From Power - Alternating Code (E-region) - 15 min
pfa20240108.001_ac_fit_15min.001.h5 	 2000215 	 Fitted - Alternating Code (E-region) - 15 min
pfa20240108.001_ac_nenotr_20min.001.h5 	 1000220 	 Ne From Power - Alternating Code (E-region) - 20 min
pfa20240108.001_ac_fit_20min.001.h5 	 2000220 	 Fitted - Alternating Code (E-region) - 20 min
pfa20240108.001_lp_nenotr_01min.001.h5 	 1000101 	 Ne From Power - Long Pulse (F-region) - 1 min
pfa20240108.001_lp_fit_01min.001.h5 	 2000101 	 Fitted - Long Pulse (F-region) - 1 min
pfa20240108.001_lp_nenotr_03min.001.h5 	 1000103 	 Ne From Power - Long Pulse (F-region) - 3 min
pfa20240108.001_lp_fit_03min.001.h5 	 2000103 	 Fitted - Long Pulse (F-region) - 3 min
pfa20240108.001_lp_nenotr_05min.001.h5 	 1000105 	 Ne From Power - Long Pulse (F-region) - 5 min
pfa20240108.001_lp_fit_05min.001.h5 	 2000105 	 Fitted - Long Pulse (F-region) - 5 min
pfa20240108.001_lp_nenotr_10min.001.h5 	 1000110 	 Ne From Power - Long Pulse (F-region) - 10 min
pfa20240108.001_lp_fit_10min.001.h5 	 2000110 	 Fitted - Long Pulse (F-region) - 10 min
pfa20240108.001_lp_nenotr_15min.001.h5 	 1000115 	 Ne From Power - Long Pulse (F-region) - 15 min
pfa20240108.001_lp_fit_15min.001.h5 	 2000115 	 Fitted - Long Pulse (F-region) - 15 min
pfa20240108.001_lp_nenotr_20min.001.h5 	 1000120 	 Ne From Power - Long Pulse (F-region) - 20 min
pfa20240108.001_lp_fit_20min.001.h5 	 2000120 	 Fitted - Long Pulse (F-region) - 20 min
pfa20240108.001_lp_vvels_01min.001.h5 	 3000101 	 Resolved Velocity - Long Pulse (F-region) - 1 min
pfa20240108.001_lp_vvels_03min.001.h5 	 3000103 	 Resolved Velocity - Long Pulse (F-region) - 3 min
pfa20240108.001_lp_vvels_05min.001.h5 	 3000105 	 Resolved Velocity - Long Pulse (F-region) - 5 min
# Download the file that we need to run these examples
os.makedirs('data', exist_ok=True)
filepath= 'data/pfa20240108.001_lp_fit_01min.001.h5 '

if not os.path.exists(filepath):

    fileList = data.getExperimentFiles(expList[0].id)
    for file0 in fileList:
        if file0.kindatdesc == 'Fitted - Long Pulse (F-region) - 1 min':
            file2download = file0.name
            break
        
    print('Downloading data file...')

    file = data.downloadFile(file2download, filepath, 
                               user_fullname, user_email, user_affiliation,'hdf5')    
    print('...Done!')

else:
    print(f"File {filepath} already downloaded")
    
File data/pfa20240108.001_lp_fit_01min.001.h5  already downloaded
h5 = h5py.File(filepath, 'r')
with h5py.File(filepath, 'r') as h5:
    print(list(h5.keys()))
    print()
    print(list(h5['Data'].keys()))
    print()
    print(list(h5['Metadata'].keys()))
    print()
    print(list(h5['Data']['Array Layout']))
['Data', 'Metadata']

['Array Layout', 'Table Layout']

['Data Parameters', 'Experiment Notes', 'Experiment Parameters', 'Independent Spatial Parameters', 'Parameters Used to Split Array Data', '_record_layout']

['Array with beamid=63149 ', 'Array with beamid=63197 ', 'Array with beamid=63239 ', 'Array with beamid=63281 ', 'Array with beamid=63317 ', 'Array with beamid=63365 ', 'Array with beamid=63401 ', 'Array with beamid=63449 ', 'Array with beamid=64016 ', 'Array with beamid=64031 ', 'Array with beamid=64046 ', 'Array with beamid=64067 ', 'Array with beamid=64157 ', 'Array with beamid=64163 ', 'Array with beamid=64205 ', 'Array with beamid=64232 ', 'Array with beamid=64247 ', 'Array with beamid=64475 ', 'Array with beamid=64817 ', 'Array with beamid=64973 ', 'Array with beamid=65057 ', 'Array with beamid=65282 ', 'Array with beamid=65363 ']
with h5py.File(filepath, 'r') as h5:
    for i in range(len(np.array(h5['Metadata/Data Parameters']))):
        print(i, h5['Metadata/Data Parameters'][i][1].decode('utf-8'), f"[{h5['Metadata/Data Parameters'][i][3].decode('utf-8')}]")
0 Year (universal time) [y]
1 Month (universal time) [m]
2 Day (universal time) [d]
3 Hour (universal time) [h]
4 Minute (universal time) [m]
5 Second (universal time) [s]
6 Logical Record Number [N/A]
7 Kind of data [N/A]
8 Instrument Code [N/A]
9 Unix seconds (1/1/1970) at start [s]
10 Unix seconds (1/1/1970) at end [s]
11 Mean azimuth angle (0=geog N;90=east) [deg]
12 Elevation angle (0=horizontal;90=vert) [deg]
13 Beam identifier [N/A]
14 Peak power [kW]
15 Average number of transmit AEUs used [N/A]
16 Average number of receive AEUs used [N/A]
17 Code baud count per pulse [N/A]
18 Pulse length [s]
19 Transmitted frequency [Hz]
20 Received frequency [Hz]
21 Range [km]
22 Electron density (Ne) [m-3]
23 Error in Electron density (Ne) [m-3]
24 Ion temperature (Ti) [K]
25 Error in Ion temperature (Ti) [K]
26 Electron temperature (Te) [K]
27 Error in Electron temperature (Te) [K]
28 Line of sight ion velocity (pos = away) [ms-1]
29 Error in Line of sight ion velocity (pos = away) [ms-1]
30 Corrected Geomagnetic latitude [deg]
31 Corrected Geomagnetic Longitude [deg]
32 Composition - [O+]/Ne [N/A]
33 Error in Composition - [O+]/Ne [N/A]
34 Reduced-chi square of fit [N/A]
35 Signal to noise ratio [N/A]

We will mostly be dealing with the Array Layout, which is organized by beam.

with h5py.File(filepath, 'r') as h5:
    print(list(h5['Data']['Array Layout']))
    print()
['Array with beamid=63149 ', 'Array with beamid=63197 ', 'Array with beamid=63239 ', 'Array with beamid=63281 ', 'Array with beamid=63317 ', 'Array with beamid=63365 ', 'Array with beamid=63401 ', 'Array with beamid=63449 ', 'Array with beamid=64016 ', 'Array with beamid=64031 ', 'Array with beamid=64046 ', 'Array with beamid=64067 ', 'Array with beamid=64157 ', 'Array with beamid=64163 ', 'Array with beamid=64205 ', 'Array with beamid=64232 ', 'Array with beamid=64247 ', 'Array with beamid=64475 ', 'Array with beamid=64817 ', 'Array with beamid=64973 ', 'Array with beamid=65057 ', 'Array with beamid=65282 ', 'Array with beamid=65363 ']
with h5py.File(filepath, 'r') as h5:
    print(list(h5['Data/Array Layout']['Array with beamid=63149 '].keys()))
    print()
    print(list(h5['Data/Array Layout']['Array with beamid=63149 ']['2D Parameters'].keys()))
    print()
    print(list(h5['Data/Array Layout']['Array with beamid=63149 ']['1D Parameters']))
    print()
['1D Parameters', '2D Parameters', 'Layout Description', 'range', 'timestamps']

['Data Parameters', 'cgm_lat', 'cgm_long', 'chisq', 'dne', 'dpo+', 'dte', 'dti', 'dvo', 'ne', 'po+', 'sn', 'te', 'ti', 'vo']

['Data Parameters', 'azm', 'beamid', 'cbadl', 'elm', 'numrxaeu', 'numtxaeu', 'pl', 'power', 'rfreq', 'tfreq']

Electron Density#

Units: m\(^{-3}\)

Dimensions: Nbeams x Nranges x Nrecords

  • Nrecords: Number of time records

  • Nbeams: Number of radar beams

  • Nranges: Number of radar range gates

with h5py.File(filepath, 'r') as h5:
    ne = []
    for key in h5['Data/Array Layout'].keys():
        ne.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['ne']))
    ne = np.array(ne)
ne.shape
(23, 73, 543)

Ion Temperature#

Units: K

Dimensions: Nbeams x Nranges x Nrecords

with h5py.File(filepath, 'r') as h5:
    ti = []
    for key in h5['Data/Array Layout'].keys():
        ti.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['ti']))
    ti = np.array(ti)
ti.shape
(23, 73, 543)

Electron Temperature#

Units: K

Dimensions: Nbeams x Nranges x Nrecords

with h5py.File(filepath, 'r') as h5:
    te = []
    for key in h5['Data/Array Layout'].keys():
        te.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['te']))
    te = np.array(te)
te.shape
(23, 73, 543)

Line-of-Sight Velocity#

Units: m/s

Dimensions: Nbeams x Nranges x Nrecords

with h5py.File(filepath, 'r') as h5:
    Vlos = []
    for key in h5['Data/Array Layout'].keys():
        Vlos.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['vo']))
    Vlos = np.array(Vlos)
Vlos.shape
(23, 73, 543)

Time#

The Time group lists timestamps in a variety of formats, but UnixTime (seconds from January 1, 1970) is often most convenient for programming purposes.

Dimensions: Nbeams x Nrecords

with h5py.File(filepath, 'r') as h5:
    time = []
    for key in h5['Data/Array Layout'].keys():
        time.append(np.array(h5['Data/Array Layout'][key]['timestamps']))
    time = np.array(time)
time.shape
(23, 543)

Range [m]#

Dimensions: Nranges

with h5py.File(filepath, 'r') as h5:
    rng = np.array(h5['Data/Array Layout']['Array with beamid=63149 ']['range'])
rng.shape
(73,)

Geographic Position [deg]#

Corrected Geomagnetic Latitude and Corrected Geomagnetic Longitude.

Dimensions: Nbeams x Nranges x Nrecords

with h5py.File(filepath, 'r') as h5:
    cgm_lat = []
    cgm_long = []
    for key in h5['Data/Array Layout'].keys():
        cgm_lat.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['cgm_lat']))
        cgm_long.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['cgm_long']))
    cgm_lat = np.array(cgm_lat)
    cgm_long = np.array(cgm_long)
    
cgm_lat.shape
(23, 73, 543)

Beam Positions#

The beam azimuths and elevations are contained in the 1D Parameters array.

Dimensions: Nbeams

with h5py.File(filepath, 'r') as h5:
    elm = []
    azm = []
    for key in h5['Data/Array Layout'].keys():
        elm.append(np.array(h5['Data/Array Layout'][key]['1D Parameters']['elm'][0]))
        azm.append(np.array(h5['Data/Array Layout'][key]['1D Parameters']['azm'][0]))
    elm = np.array(elm)
    azm = np.array(azm)
elm.shape  
(23,)

Site Info#

Various information about the radar site and its coordinates are contained within the Site group.

with h5py.File(filepath, 'r') as h5:
    param = list(h5['Metadata']['Experiment Parameters'])

print('Site Name:', param[0][1].decode('utf-8'))
print('Instrument Latitude:', param[8][1].decode('utf-8'))
print('Instrument Longitude:', param[9][1].decode('utf-8'))
print('Instrument Altitude:', param[10][1].decode('utf-8'))
Site Name: Poker Flat IS Radar
Instrument Latitude: 65.13
Instrument Longitude: 212.529
Instrument Altitude: 0.215
# Close h5 object
h5.close()