Common Data Fields#
This notebook will walk through how to access some of the more common data fields from the Madrigal hdf5 data files. This should be treated as a quick reference for common scientific applications and is NOT intended to be a comprehensive map of everything in these files. Refer to metadata contained within the files for descriptions of other arrays.
import h5py
import madrigalWeb.madrigalWeb
import os
import numpy as np
madrigalUrl='http://cedar.openmadrigal.org'
data = madrigalWeb.madrigalWeb.MadrigalData(madrigalUrl)
user_fullname = 'Student Example'
user_email = 'isr.summer.school@gmail.com'
user_affiliation= 'ISR Summer School 2024'
code = 61 # PFISR
year = 2024
month = 1
day = 8
hour1 = 7
minute1 = 1
hour2 = 13
# list of experiments inside a time period of a day
expList = data.getExperiments(code,year,month,day,hour1,minute1,0,year,month,day,hour2,0,0)
for exp in expList:
print(str(exp))
id: 100278619
realUrl: http://cedar.openmadrigal.org/showExperiment/?experiment_list=100278619
url: http://cedar.openmadrigal.org/madtoc/experiments4/2024/pfa/08jan24a
name: Themis36 - Auroral and convection measurements
siteid: 10
sitename: CEDAR
instcode: 61
instname: Poker Flat IS Radar
startyear: 2024
startmonth: 1
startday: 8
starthour: 7
startmin: 1
startsec: 4
endyear: 2024
endmonth: 1
endday: 8
endhour: 18
endmin: 0
endsec: 0
isLocal: True
madrigalUrl: http://cedar.openmadrigal.org/
PI: Asti Bhatt
PIEmail: asti.bhatt@sri.com
uttimestamp: 1709109883
access: 2
Madrigal version: 3.2
fileList = data.getExperimentFiles(expList[0].id)
for file0 in fileList:
print(os.path.basename(file0.name),'\t', file0.kindat, '\t',file0.kindatdesc)
pfa20240108.001_ac_nenotr_01min.001.h5 1000201 Ne From Power - Alternating Code (E-region) - 1 min
pfa20240108.001_ac_fit_01min.001.h5 2000201 Fitted - Alternating Code (E-region) - 1 min
pfa20240108.001_ac_nenotr_03min.001.h5 1000203 Ne From Power - Alternating Code (E-region) - 3 min
pfa20240108.001_ac_fit_03min.001.h5 2000203 Fitted - Alternating Code (E-region) - 3 min
pfa20240108.001_ac_nenotr_05min.001.h5 1000205 Ne From Power - Alternating Code (E-region) - 5 min
pfa20240108.001_ac_fit_05min.001.h5 2000205 Fitted - Alternating Code (E-region) - 5 min
pfa20240108.001_ac_nenotr_10min.001.h5 1000210 Ne From Power - Alternating Code (E-region) - 10 min
pfa20240108.001_ac_fit_10min.001.h5 2000210 Fitted - Alternating Code (E-region) - 10 min
pfa20240108.001_ac_nenotr_15min.001.h5 1000215 Ne From Power - Alternating Code (E-region) - 15 min
pfa20240108.001_ac_fit_15min.001.h5 2000215 Fitted - Alternating Code (E-region) - 15 min
pfa20240108.001_ac_nenotr_20min.001.h5 1000220 Ne From Power - Alternating Code (E-region) - 20 min
pfa20240108.001_ac_fit_20min.001.h5 2000220 Fitted - Alternating Code (E-region) - 20 min
pfa20240108.001_lp_nenotr_01min.001.h5 1000101 Ne From Power - Long Pulse (F-region) - 1 min
pfa20240108.001_lp_fit_01min.001.h5 2000101 Fitted - Long Pulse (F-region) - 1 min
pfa20240108.001_lp_nenotr_03min.001.h5 1000103 Ne From Power - Long Pulse (F-region) - 3 min
pfa20240108.001_lp_fit_03min.001.h5 2000103 Fitted - Long Pulse (F-region) - 3 min
pfa20240108.001_lp_nenotr_05min.001.h5 1000105 Ne From Power - Long Pulse (F-region) - 5 min
pfa20240108.001_lp_fit_05min.001.h5 2000105 Fitted - Long Pulse (F-region) - 5 min
pfa20240108.001_lp_nenotr_10min.001.h5 1000110 Ne From Power - Long Pulse (F-region) - 10 min
pfa20240108.001_lp_fit_10min.001.h5 2000110 Fitted - Long Pulse (F-region) - 10 min
pfa20240108.001_lp_nenotr_15min.001.h5 1000115 Ne From Power - Long Pulse (F-region) - 15 min
pfa20240108.001_lp_fit_15min.001.h5 2000115 Fitted - Long Pulse (F-region) - 15 min
pfa20240108.001_lp_nenotr_20min.001.h5 1000120 Ne From Power - Long Pulse (F-region) - 20 min
pfa20240108.001_lp_fit_20min.001.h5 2000120 Fitted - Long Pulse (F-region) - 20 min
pfa20240108.001_lp_vvels_01min.001.h5 3000101 Resolved Velocity - Long Pulse (F-region) - 1 min
pfa20240108.001_lp_vvels_03min.001.h5 3000103 Resolved Velocity - Long Pulse (F-region) - 3 min
pfa20240108.001_lp_vvels_05min.001.h5 3000105 Resolved Velocity - Long Pulse (F-region) - 5 min
# Download the file that we need to run these examples
os.makedirs('data', exist_ok=True)
filepath= 'data/pfa20240108.001_lp_fit_01min.001.h5 '
if not os.path.exists(filepath):
fileList = data.getExperimentFiles(expList[0].id)
for file0 in fileList:
if file0.kindatdesc == 'Fitted - Long Pulse (F-region) - 1 min':
file2download = file0.name
break
print('Downloading data file...')
file = data.downloadFile(file2download, filepath,
user_fullname, user_email, user_affiliation,'hdf5')
print('...Done!')
else:
print(f"File {filepath} already downloaded")
File data/pfa20240108.001_lp_fit_01min.001.h5 already downloaded
h5 = h5py.File(filepath, 'r')
with h5py.File(filepath, 'r') as h5:
print(list(h5.keys()))
print()
print(list(h5['Data'].keys()))
print()
print(list(h5['Metadata'].keys()))
print()
print(list(h5['Data']['Array Layout']))
['Data', 'Metadata']
['Array Layout', 'Table Layout']
['Data Parameters', 'Experiment Notes', 'Experiment Parameters', 'Independent Spatial Parameters', 'Parameters Used to Split Array Data', '_record_layout']
['Array with beamid=63149 ', 'Array with beamid=63197 ', 'Array with beamid=63239 ', 'Array with beamid=63281 ', 'Array with beamid=63317 ', 'Array with beamid=63365 ', 'Array with beamid=63401 ', 'Array with beamid=63449 ', 'Array with beamid=64016 ', 'Array with beamid=64031 ', 'Array with beamid=64046 ', 'Array with beamid=64067 ', 'Array with beamid=64157 ', 'Array with beamid=64163 ', 'Array with beamid=64205 ', 'Array with beamid=64232 ', 'Array with beamid=64247 ', 'Array with beamid=64475 ', 'Array with beamid=64817 ', 'Array with beamid=64973 ', 'Array with beamid=65057 ', 'Array with beamid=65282 ', 'Array with beamid=65363 ']
with h5py.File(filepath, 'r') as h5:
for i in range(len(np.array(h5['Metadata/Data Parameters']))):
print(i, h5['Metadata/Data Parameters'][i][1].decode('utf-8'), f"[{h5['Metadata/Data Parameters'][i][3].decode('utf-8')}]")
0 Year (universal time) [y]
1 Month (universal time) [m]
2 Day (universal time) [d]
3 Hour (universal time) [h]
4 Minute (universal time) [m]
5 Second (universal time) [s]
6 Logical Record Number [N/A]
7 Kind of data [N/A]
8 Instrument Code [N/A]
9 Unix seconds (1/1/1970) at start [s]
10 Unix seconds (1/1/1970) at end [s]
11 Mean azimuth angle (0=geog N;90=east) [deg]
12 Elevation angle (0=horizontal;90=vert) [deg]
13 Beam identifier [N/A]
14 Peak power [kW]
15 Average number of transmit AEUs used [N/A]
16 Average number of receive AEUs used [N/A]
17 Code baud count per pulse [N/A]
18 Pulse length [s]
19 Transmitted frequency [Hz]
20 Received frequency [Hz]
21 Range [km]
22 Electron density (Ne) [m-3]
23 Error in Electron density (Ne) [m-3]
24 Ion temperature (Ti) [K]
25 Error in Ion temperature (Ti) [K]
26 Electron temperature (Te) [K]
27 Error in Electron temperature (Te) [K]
28 Line of sight ion velocity (pos = away) [ms-1]
29 Error in Line of sight ion velocity (pos = away) [ms-1]
30 Corrected Geomagnetic latitude [deg]
31 Corrected Geomagnetic Longitude [deg]
32 Composition - [O+]/Ne [N/A]
33 Error in Composition - [O+]/Ne [N/A]
34 Reduced-chi square of fit [N/A]
35 Signal to noise ratio [N/A]
We will mostly be dealing with the Array Layout, which is organized by beam.
with h5py.File(filepath, 'r') as h5:
print(list(h5['Data']['Array Layout']))
print()
['Array with beamid=63149 ', 'Array with beamid=63197 ', 'Array with beamid=63239 ', 'Array with beamid=63281 ', 'Array with beamid=63317 ', 'Array with beamid=63365 ', 'Array with beamid=63401 ', 'Array with beamid=63449 ', 'Array with beamid=64016 ', 'Array with beamid=64031 ', 'Array with beamid=64046 ', 'Array with beamid=64067 ', 'Array with beamid=64157 ', 'Array with beamid=64163 ', 'Array with beamid=64205 ', 'Array with beamid=64232 ', 'Array with beamid=64247 ', 'Array with beamid=64475 ', 'Array with beamid=64817 ', 'Array with beamid=64973 ', 'Array with beamid=65057 ', 'Array with beamid=65282 ', 'Array with beamid=65363 ']
with h5py.File(filepath, 'r') as h5:
print(list(h5['Data/Array Layout']['Array with beamid=63149 '].keys()))
print()
print(list(h5['Data/Array Layout']['Array with beamid=63149 ']['2D Parameters'].keys()))
print()
print(list(h5['Data/Array Layout']['Array with beamid=63149 ']['1D Parameters']))
print()
['1D Parameters', '2D Parameters', 'Layout Description', 'range', 'timestamps']
['Data Parameters', 'cgm_lat', 'cgm_long', 'chisq', 'dne', 'dpo+', 'dte', 'dti', 'dvo', 'ne', 'po+', 'sn', 'te', 'ti', 'vo']
['Data Parameters', 'azm', 'beamid', 'cbadl', 'elm', 'numrxaeu', 'numtxaeu', 'pl', 'power', 'rfreq', 'tfreq']
Electron Density#
Units: m\(^{-3}\)
Dimensions: Nbeams x Nranges x Nrecords
Nrecords: Number of time records
Nbeams: Number of radar beams
Nranges: Number of radar range gates
with h5py.File(filepath, 'r') as h5:
ne = []
for key in h5['Data/Array Layout'].keys():
ne.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['ne']))
ne = np.array(ne)
ne.shape
(23, 73, 543)
Ion Temperature#
Units: K
Dimensions: Nbeams x Nranges x Nrecords
with h5py.File(filepath, 'r') as h5:
ti = []
for key in h5['Data/Array Layout'].keys():
ti.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['ti']))
ti = np.array(ti)
ti.shape
(23, 73, 543)
Electron Temperature#
Units: K
Dimensions: Nbeams x Nranges x Nrecords
with h5py.File(filepath, 'r') as h5:
te = []
for key in h5['Data/Array Layout'].keys():
te.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['te']))
te = np.array(te)
te.shape
(23, 73, 543)
Line-of-Sight Velocity#
Units: m/s
Dimensions: Nbeams x Nranges x Nrecords
with h5py.File(filepath, 'r') as h5:
Vlos = []
for key in h5['Data/Array Layout'].keys():
Vlos.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['vo']))
Vlos = np.array(Vlos)
Vlos.shape
(23, 73, 543)
Time#
The Time group lists timestamps in a variety of formats, but UnixTime (seconds from January 1, 1970) is often most convenient for programming purposes.
Dimensions: Nbeams x Nrecords
with h5py.File(filepath, 'r') as h5:
time = []
for key in h5['Data/Array Layout'].keys():
time.append(np.array(h5['Data/Array Layout'][key]['timestamps']))
time = np.array(time)
time.shape
(23, 543)
Range [m]#
Dimensions: Nranges
with h5py.File(filepath, 'r') as h5:
rng = np.array(h5['Data/Array Layout']['Array with beamid=63149 ']['range'])
rng.shape
(73,)
Geographic Position [deg]#
Corrected Geomagnetic Latitude and Corrected Geomagnetic Longitude.
Dimensions: Nbeams x Nranges x Nrecords
with h5py.File(filepath, 'r') as h5:
cgm_lat = []
cgm_long = []
for key in h5['Data/Array Layout'].keys():
cgm_lat.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['cgm_lat']))
cgm_long.append(np.array(h5['Data/Array Layout'][key]['2D Parameters']['cgm_long']))
cgm_lat = np.array(cgm_lat)
cgm_long = np.array(cgm_long)
cgm_lat.shape
(23, 73, 543)
Beam Positions#
The beam azimuths and elevations are contained in the 1D Parameters array.
Dimensions: Nbeams
with h5py.File(filepath, 'r') as h5:
elm = []
azm = []
for key in h5['Data/Array Layout'].keys():
elm.append(np.array(h5['Data/Array Layout'][key]['1D Parameters']['elm'][0]))
azm.append(np.array(h5['Data/Array Layout'][key]['1D Parameters']['azm'][0]))
elm = np.array(elm)
azm = np.array(azm)
elm.shape
(23,)
Site Info#
Various information about the radar site and its coordinates are contained within the Site group.
with h5py.File(filepath, 'r') as h5:
param = list(h5['Metadata']['Experiment Parameters'])
print('Site Name:', param[0][1].decode('utf-8'))
print('Instrument Latitude:', param[8][1].decode('utf-8'))
print('Instrument Longitude:', param[9][1].decode('utf-8'))
print('Instrument Altitude:', param[10][1].decode('utf-8'))
Site Name: Poker Flat IS Radar
Instrument Latitude: 65.13
Instrument Longitude: 212.529
Instrument Altitude: 0.215
# Close h5 object
h5.close()