Python 3 Support and Docker Update

.travis.yml

@@ -4,8 +4,10 @@ language: python
 
 python:
     - 2.7
+    - 3.6
     # PyPy versions
     - pypy2.7-6.0
+    - pypy3
 
 services:
     - docker
@@ -27,6 +29,7 @@ jobs:
           python: 2.7
           script:
             - make docker-cpu
+            - make docker
             - bash ./.travis_deploy_docs.sh
 
 script:
@@ -38,7 +41,7 @@ script:
         simplejson \
         pint \
         graphviz \
-        git+https://github.com/davidjamesca/ctypesgen.git@3d2d9803339503d2988382aa861b47a6a4872c32 \
+        git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f \
         coveralls \
         codecov
     - sudo make -j NOCUDA=1

Dockerfile.cpu

@@ -6,6 +6,13 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 ENV TERM xterm
 
+# Update ctypesgen
+RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py && \
+    rm get-pip.py
+RUN pip --no-cache-dir install \
+        git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f
+
 # Build the library
 WORKDIR /bifrost
 COPY . .

Dockerfile.gpu

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:8.0
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
 
 MAINTAINER Ben Barsdell <[email protected]>
 
@@ -31,7 +31,7 @@ RUN pip --no-cache-dir install \
         contextlib2 \
         simplejson \
         pint \
-        git+https://github.com/davidjamesca/ctypesgen.git@3d2d9803339503d2988382aa861b47a6a4872c32 \
+        git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f \
         graphviz
 
 ENV TERM xterm

Dockerfile_prereq.gpu

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:8.0
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
 
 MAINTAINER Ben Barsdell <[email protected]>
 
@@ -31,7 +31,7 @@ RUN pip --no-cache-dir install \
         contextlib2 \
         simplejson \
         pint \
-        git+https://github.com/davidjamesca/ctypesgen.git@3d2d9803339503d2988382aa861b47a6a4872c32 \
+        git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f \
         graphviz
 
 ENV TERM xterm

README.md

@@ -99,7 +99,7 @@ print "All done"
  * ctypesgen
 
 ```
-$ sudo pip install numpy contextlib2 pint git+https://github.com/davidjamesca/ctypesgen.git@3d2d9803339503d2988382aa861b47a6a4872c32
+$ sudo pip install numpy contextlib2 pint git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f
 ```
 
 ### Bifrost installation

docs/source/Getting-started-guide.rst

@@ -44,7 +44,7 @@ numpy, matplotlib, contextlib2, simplejson, pint, graphviz, ctypesgen
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 If you have already installed pip, this step should be as simple as
-``pip install --user numpy matplotlib contextlib2 simplejson pint graphviz git+https://github.com/davidjamesca/ctypesgen.git@3d2d9803339503d2988382aa861b47a6a4872c32``.
+``pip install --user numpy matplotlib contextlib2 simplejson pint graphviz git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f``.
 
 C++ dependencies
 ~~~~~~~~~~~~~~~~

python/Makefile

@@ -21,7 +21,7 @@ $(BIFROST_PYTHON_VERSION_FILE): ../config.mk
 	@echo "__version__ = \"$(LIBBIFROST_MAJOR).$(LIBBIFROST_MINOR).$(LIBBIFROST_PATCH)\"" > $@
 
 define run_ctypesgen
-	ctypesgen.py -l$1 -I$2 $^ -o $@
+	python -c 'from ctypesgen import main as ctypeswrap; ctypeswrap.main()' -l$1 -I$2 $^ -o $@
 	# WAR for 'const char**' being generated as POINTER(POINTER(c_char)) instead of POINTER(c_char_p)
 	sed -i 's/POINTER(c_char)/c_char_p/g' $@
 	# WAR for a buggy WAR in ctypesgen that breaks type checking and auto-byref functionality

python/bifrost/DataType.py

@@ -38,6 +38,10 @@
 cf32: 32+32-bit complex floating point
 """
 
+import sys
+if sys.version_info > (3,):
+    xrange = range
+
 from libbifrost import _bf
 import numpy as np
 

python/bifrost/__init__.py

@@ -31,6 +31,9 @@
 """
 
 # TODO: Decide how to organise the namespace
+
+from __future__ import print_function
+
 import core, memory, affinity, ring, block, address, udp_socket
 import pipeline
 import device
@@ -49,9 +52,9 @@
 try:
     from .version import __version__
 except ImportError:
-    print "*************************************************************************"
-    print "Please run `make` from the root of the source tree to generate version.py"
-    print "*************************************************************************"
+    print("*************************************************************************")
+    print("Please run `make` from the root of the source tree to generate version.py")
+    print("*************************************************************************")
     raise
 __author__     = "The Bifrost Authors"
 __copyright__  = "Copyright (c) 2016, The Bifrost Authors. All rights reserved.\nCopyright (c) 2016, NVIDIA CORPORATION. All rights reserved."

python/bifrost/addon/leda/bandfiles.py

@@ -24,6 +24,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from __future__ import print_function
+
 import os, sys
 
 sys.path.append('..')
@@ -83,12 +85,12 @@ def __init__(self, fname):
     obs1 = extract_obs_offset_from_name(fname)
     obs2 = extract_obs_offset_in_file(fname)
     if obs1 != obs2 and obs1 != "UNKNOWN" and obs2 !="UNKNOWN":
-      print "Consistency Error", fname, ": OBS_OFFSET in file doesn't match the offset in the name"
+      print("Consistency Error", fname, ": OBS_OFFSET in file doesn't match the offset in the name")
     """
 
     if hedr["SOURCE"] == "LEDA_TEST":
-      if not is_integer(n_scans): print "CONSISTENCY ERROR, ", fname, "scan:",n_scans, "is not integer"
-      if not is_integer((self.end_time-self.start_time)/9.0): print "CONSISTENCY ERROR", fname, ": not 9 sec dump in file"
+      if not is_integer(n_scans): print("CONSISTENCY ERROR, ", fname, "scan:",n_scans, "is not integer")
+      if not is_integer((self.end_time-self.start_time)/9.0): print("CONSISTENCY ERROR", fname, ": not 9 sec dump in file")
 
 # Gather the file info for all files that have different frequency but the same observation time.
 class BandFiles(object):
@@ -106,7 +108,7 @@ def __init__(self, basename):
     if basename[-5:] == ".dada":                # Just a single file
       if os.access(basename,os.R_OK): self.files.append(FileInfo(basename))
       else: "Error:", basename, "does not exist or is not readable"
-      print basename, FileInfo(basename), self.files
+      print(basename, FileInfo(basename), self.files)
     else:
 
       # Look for files in all the standard locations
@@ -138,7 +140,7 @@ def __init__(self, basename):
       if f.start_time not in self.start_time_present: self.start_time_present.append(f.start_time)
 
     if len(self.start_time_present) > 1:
-      print "Error: Files with same timestamp in their name have different internal time. Basename:",basename
+      print("Error: Files with same timestamp in their name have different internal time. Basename:",basename)
       #sys.exit(1)
 
     self.start_time = self.start_time_present[0]

python/bifrost/addon/leda/blocks.py

@@ -28,6 +28,11 @@
 This file contains blocks specific to LEDA-OVRO.
 """
 
+from __future__ import print_function
+import sys
+if sys.version_info > (3,):
+    xrange = range
+
 import os
 import bandfiles
 import bifrost
@@ -75,10 +80,10 @@ def __init__(self, time_stamp, core=-1, gulp_nframe=4096):
         i += 1
 
     # Report what we've got
-    print "Num files in time:",  len(beamformer_scans)
-    print "File and number:"
+    print("Num files in time:",  len(beamformer_scans))
+    print("File and number:")
     for scan in beamformer_scans:
-      print os.path.basename(scan.files[0].name)+":", len(scan.files)
+      print(os.path.basename(scan.files[0].name)+":", len(scan.files))
 
     self.beamformer_scans = beamformer_scans     # List of full-band time steps
 
@@ -100,7 +105,7 @@ def main(self, input_rings, output_rings):
     ohdr["tsamp"] = self.SAMPLING_RATE
     ohdr['foff'] = self.CHANNEL_WIDTH
 
-    #print length_one_second, ring_span_size, file_chunk_size, number_of_chunks
+    #print(length_one_second, ring_span_size, file_chunk_size, number_of_chunks)
 
     with self.oring.begin_writing() as oring:
 
@@ -109,7 +114,7 @@ def main(self, input_rings, output_rings):
         # Go through the frequencies
         for f in scan.files:
 
-          print "Opening", f.name
+          print("Opening", f.name)
 
           with open(f.name,'rb') as ifile:
             ifile.read(self.HEADER_SIZE)
@@ -120,16 +125,16 @@ def main(self, input_rings, output_rings):
             self.oring.resize(ring_span_size)
             with oring.begin_sequence(f.name, header=json.dumps(ohdr)) as osequence:
 
-              for i in range(number_of_seconds):
+              for i in xrange(number_of_seconds):
                   # Get a chunk of data from the file. The whole band is used, but only a chunk of time (1 second).
                   # Massage the data so it can go through the ring. That means changng the data type and flattening.
                 try:
                     data = np.fromfile(ifile, count=file_chunk_size, dtype=np.int8).astype(np.float32)
                 except:
-                    print "Bad read. Stopping read."
+                    print("Bad read. Stopping read.")
                     return
                 if data.size != length_one_second*self.N_BEAM*self.N_CHAN*2:
-                    print "Bad data shape. Stopping read."
+                    print("Bad data shape. Stopping read.")
                     return
                 data = data.reshape(length_one_second, self.N_BEAM, self.N_CHAN, 2)
                 power = (data[...,0]**2 + data[...,1]**2).mean(axis=1)  # Now have time by frequency.

python/bifrost/addon/leda/make_header.py

@@ -36,6 +36,8 @@
 Makes header.txt files that is used by corr2uvfit and DuCT.
 """
 
+from __future__ import print_function
+
 import numpy as np
 import os, sys, ephem, datetime
 from dateutil import tz
@@ -63,7 +65,7 @@ def __init__(self, filename, warnings, file_size):
         self.filename = filename
     self.warnings = warnings
      self.file_size = file_size    # Externally supplied
-    #print filename, warnings, file_size
+    #print(filename, warnings, file_size)
     self.generate_info()
 
     def generate_info(self):
@@ -113,7 +115,7 @@ def generate_info(self):
         data_size_dsk = int(header["FILE_SIZE"])    # these data sizes don't include header
         data_size_hdr = data_size_dsk
       else:                    # Failure
-        if self.warnings: print "WARNING: File is zipped and FILE_SIZE is not in header and file_size not supplied. "
+        if self.warnings: print("WARNING: File is zipped and FILE_SIZE is not in header and file_size not supplied. ")
         have_size = False
         data_size_hdr = data_size_dsk = 0
     else:                     # File not zipped. Can get true complete file size
@@ -122,7 +124,7 @@ def generate_info(self):
       else: data_size_hdr = data_size_dsk
 
         if data_size_hdr != data_size_dsk:
-      if self.warnings: print "WARNING: Data size in file doesn't match actual size. Using actual size."
+      if self.warnings: print("WARNING: Data size in file doesn't match actual size. Using actual size.")
 
     data_size = data_size_dsk        # Settle on this as the size of the data
 
@@ -135,13 +137,13 @@ def generate_info(self):
 
         if "BYTES_PER_AVG" in header and have_size:
        if data_size % bpa != 0:
-        if self.warnings: print "WARNING: BYTES_PER_AVG does not result in an integral number of scans"
+        if self.warnings: print("WARNING: BYTES_PER_AVG does not result in an integral number of scans")
         if "DATA_ORDER" in header and self.data_order == 'TIME_SUBSET_CHAN_TRIANGULAR_POL_POL_COMPLEX':
           if self.warnings: 
-        print 'DATA_ORDER is TIME_SUBSET_CHAN_TRIANGULAR_POL_POL_COMPLEX, resetting BYTES_PER_AVG to',(109*32896*2*2+9*109*1270*2*2)*8,"(fixed)"
+        print('DATA_ORDER is TIME_SUBSET_CHAN_TRIANGULAR_POL_POL_COMPLEX, resetting BYTES_PER_AVG to',(109*32896*2*2+9*109*1270*2*2)*8,"(fixed)")
           bpa = (109*32896*2*2+9*109*1270*2*2)*8
           if data_size % bpa != 0 and self.warnings:
-            print "WARNING: BYTES_PER_AVG still doesn't give integral number of scans"
+            print("WARNING: BYTES_PER_AVG still doesn't give integral number of scans")
 
           self.n_int = float(data_size) / bpa
 
@@ -213,9 +215,9 @@ def __init__(self, header):
     ra, dec = ovro.radec_of(0, np.pi/2)
     self.lst_str = str(float(ra) / 2 / np.pi * 24)
     self.dec_str = str(float(repr(dec))*180/np.pi)
-    #print ("UTC START:   %s"%dada_file.datestamp)
-    #print ("TIME OFFSET: %s"%datetime.timedelta(seconds=dada_file.t_offset))
-    #print ("NEW START:   (%s, %s)"%(date_str, time_str))
+    #print("UTC START:   %s"%dada_file.datestamp)
+    #print("TIME OFFSET: %s"%datetime.timedelta(seconds=dada_file.t_offset))
+    #print("NEW START:   (%s, %s)"%(date_str, time_str))
 
 
 def make_header(filename, write=True, warn=True, size=None):
@@ -286,5 +288,5 @@ def make_header(filename, write=True, warn=True, size=None):
   if len(sys.argv) == 2: make_header(sys.argv[1])
   elif len(sys.argv) == 3: make_header(sys.argv[1],size=sys.argv[2])
   else:
-    print "Expecting file name and optionally file size"
+    print("Expecting file name and optionally file size")