add macro that counts number of elements in a dataset

Author: Bruno Alves

pre_processing/count_dataset_elements.py

@@ -0,0 +1,24 @@
+import tensorflow as tf
+from src.data.tfrecords import read_spectra_data as read_data
+
+batch_size = 1
+files_path = '/fred/oz012/Bruno/data/spectra/qso_zWarning/'
+dataset = read_data(files_path+'spectra2_.tfrecord', 3500) #reads all shards: spectra2_0.tfrecord, ...
+dataset = dataset.repeat(1).batch(batch_size)
+
+iterator = dataset.make_initializable_iterator()
+next_element = iterator.get_next()
+
+counter = 0
+with tf.Session() as sess:
+    sess.run(iterator.initializer)
+    while True:
+        try:
+            inputs, *params = sess.run(next_element)
+            counter += 1
+            if counter%10000==0:
+                print(counter)
+        except tf.errors.OutOfRangeError:
+            print('Number of elements inside the dataset: ', counter)
+            break
+

pre_processing/spectra_visualize_tfrecord.py

@@ -1,35 +1,30 @@
 from astropy.io import fits
 import tensorflow as tf
 import numpy as np
-from src.data.data import read_spectra_data as read_data
+from src.data.tfrecords import read_spectra_data as read_data
 from src.utilities import PlotGenSamples
 
 def _plot(data, params_data, name, n=5):
     p = PlotGenSamples(nrows=n, ncols=1)
     p.plot_spectra(data[:n], params_data[0][:n], name)
 
-batch_size = 512
-dataset_size = 255483
-files_path = '/fred/oz012/Bruno/data/spectra/boss/cmass/'
-dataset = read_data(files_path+'spectra.tfrecord', 3500)
+batch_size = 1
+dataset_size = 22791
+files_path = '/fred/oz012/Bruno/data/spectra/qso_zWarning/'
+dataset = read_data(files_path+'spectra2_.tfrecord', 3500)
 dataset = dataset.repeat(1).batch(batch_size)
 nbatches = int(np.ceil(dataset_size/batch_size))
 
 iterator = dataset.make_initializable_iterator()
 next_element = iterator.get_next()
 
+counter = 0
 with tf.Session() as sess:
     sess.run(iterator.initializer)
-    for item in range(5):
+    for item in range(2):
         inputs, *params = sess.run(next_element)
+        counter += 1
+        if counter%10000==0:
+            print(counter)
         _plot(inputs, params, name='tfrecord_data_'+str(item))
 
-"""
-local_path = '/fred/oz012/Bruno/data/spectra/boss/loz/7415/spec-7415-57097-0197.fits'
-with fits.open(local_path) as hdu:
-    flux_ = hdu[1].data['flux'].astype(np.float32)
-    lam_ = np.power( 10, hdu[1].data['loglam'] ).astype(np.float32)
-                       
-    print(flux_)
-    print(min(flux_))
-"""