Merge branch 'master' into disk-backed-ring

Author: jaycedowell

.lgtm.yml

@@ -0,0 +1,10 @@
+extraction:
+  cpp:
+    prepare:
+      packages:
+        - exuberant-ctags
+    before_index:
+      - export NOCUDA=1
+    index:
+      build_command:
+        - make -e libbifrost

.travis.yml

@@ -1,4 +1,6 @@
-sudo: required
+dist: bionic
+
+dist: bionic
 
 language: python
 
@@ -14,8 +16,10 @@ services:
 
 addons:
     apt:
+        update: true
         packages:
             - build-essential
+            - ca-certificates
             - curl
             - git
             - pkg-config
@@ -32,6 +36,9 @@ jobs:
             - make docker-cpu
             - make docker
             - bash ./.travis_deploy_docs.sh
+    allow_failures:
+        - python: 2.7
+        - python: pypy2.7-6.0
 
 script:
     - sudo pip --no-cache-dir install \

CHANGELOG

@@ -0,0 +1,20 @@
+0.9.1
+ * Fixed a problem with like_bmon.py crashing when there are a large number of pipelines
+ * Added a CHANGELOG file
+ * Added a lightweight telemetry client for the Python API
+
+0.9.0
+ * Added support for Python3
+ * Migrate from getopt to argparse for the scripts in tools
+ * Improve the test suite coverage
+ * Fixed a problem with proclog cleanup on multi-user systems
+ * Added a new bifrost.romein module for gridding data
+
+0.8.0
+ * Switched from PyCLibrary to ctypesgen for Python bindings
+ * Added tutorials for bifrost.map and bifrost.views to the documentation
+ * Switched to pretty print for bifrost.blocks.print_header
+ * Added new benchmarking tests
+ 
+0.7.0
+ * Initial release

README.md

@@ -1,8 +1,8 @@
 # Bifrost
 
-| **`CPU build`** | **`Coverage`** | 
-|----------------------|----------------------|
-|[![Travis](https://travis-ci.org/ledatelescope/bifrost.svg?branch=master)](https://travis-ci.org/ledatelescope/bifrost) | [![Coverage Status](https://coveralls.io/repos/github/ledatelescope/bifrost/badge.svg)](https://coveralls.io/github/ledatelescope/bifrost) |
+| **`CPU Build`** | **`GPU Build`** | **`Coverage`** | 
+|-----------------|-----------------|----------------|
+|[![Travis](https://travis-ci.com/ledatelescope/bifrost.svg?branch=master)](https://travis-ci.com/ledatelescope/bifrost) | [![Build Status](https://fornax.phys.unm.edu/jenkins/buildStatus/icon?job=Bifrost)](https://fornax.phys.unm.edu/jenkins/job/Bifrost/) | [![Coverage Status](https://coveralls.io/repos/github/ledatelescope/bifrost/badge.svg)](https://coveralls.io/github/ledatelescope/bifrost) |
 
 A stream processing framework for high-throughput applications.
 
@@ -11,7 +11,7 @@ A stream processing framework for high-throughput applications.
 ### [Bifrost Documentation](http://ledatelescope.github.io/bifrost/)
 ### [Bifrost Roadmap](ROADMAP.md)
 
-## A simple pipeline
+## A Simple Pipeline
 
 Here's a snippet that reads Sigproc filterbank files, applies a
 Fast Dispersion Measure Transform (FDMT) on the GPU, and writes
@@ -36,7 +36,7 @@ bf.get_default_pipeline().run()
 print "All done"
 ```
 
-## A more complex pipeline
+## A More Complex Pipeline
 
 Below is a longer snippet that demonstrates some additional features
 of Bifrost pipelines, including the BlockChainer tool, block scopes,
@@ -73,7 +73,7 @@ pipeline.run()
 print "All done"
 ```
 
-## Feature overview
+## Feature Overview
 
  - Designed for sustained high-throughput stream processing
  - Python API wraps fast C++/CUDA backend
@@ -87,11 +87,11 @@ print "All done"
 
 ## Installation
 
-### C dependencies
+### C Dependencies
 
     $ sudo apt-get install exuberant-ctags
 
-### Python dependencies
+### Python Dependencies
 
  * numpy
  * contextlib2
@@ -102,7 +102,7 @@ print "All done"
 $ sudo pip install numpy contextlib2 pint git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f
 ```
 
-### Bifrost installation
+### Bifrost Installation
 
 Edit **user.mk** to suit your system, then run:
 
@@ -116,7 +116,7 @@ You can call the following for a local Python installation:
 
     $ sudo make install PYINSTALLFLAGS="--prefix=$HOME/usr/local"
 
-### Docker container
+### Docker Container
 
 Install dependencies:
 
@@ -136,7 +136,7 @@ For CPU-only builds:
     $ make docker-cpu
     $ docker run --rm -it ledatelescope/bifrost
 
-### Running tests
+### Running Tests
 
 To run all CPU and GPU tests:
 
@@ -146,7 +146,7 @@ To run all CPU and GPU tests:
 
 ### [Online Bifrost Documentation](http://ledatelescope.github.io/bifrost/)
 
-### Building the docs with Docker
+### Building the Docs with Docker
 
 To quickly build the docs using Docker, ensure that you have
 built a Bifrost container as `ledatelescope/bifrost`.
@@ -156,7 +156,7 @@ run it, and have it complete the docs-building process for you,
 outputting the entire html documentation inside `docs/html` on
 your machine.
 
-### Building the docs from scratch
+### Building the Docs from Scratch
 
 Install sphinx and breathe using pip, and also install Doxygen.
 
@@ -171,6 +171,48 @@ by running
 
 inside the /docs directory.
 
+## Telemetry
+
+By default Bifrost installs with basic Python telemetry enabled in
+order to help inform how the software is used and to help inform future 
+development.  The data collected as part of this consist seven things:
+ * a timestamp for when the report is generated,
+ * a unique installation identifier,
+ * the Bifrost version being used, 
+ * the execution time of the Python process that imports Bifrost,
+ * which Bifrost modules are imported,
+ * which Bifrost functions are used and their average execution times, and
+ * which Bifrost scripts are used.
+These data are sent to the Bifrost developers using a HTTP POST request where
+they are aggregated.
+
+Users can opt out of telemetry collection using:
+
+```python
+from bifrost import telemetry
+telemetry.disable()
+```
+
+or by using the included `bifrost_telemetry.py` script:
+
+```
+python bifrost_telemetry.py --disable
+```
+
+This command will set a disk-based flag that disables the reporting process.
+
+## Acknowledgement
+
+If you make use of Bifrost as part of your data collection or analysis please
+include the following acknowledgement in your publications:
+
+> This research has made use of Bifrost (Cranmer et al. 2017).  Continued
+> development of Bifrost is supported by NSF award OAC/2103707.
+
+and cite:
+
+> \bibitem[Cranmer et al.(2017)]{2017JAI.....650007C} Cranmer, M.~D., Barsdell, B.~R., Price, D.~C., et al.\ 2017, Journal of Astronomical Instrumentation, 6, 1750007. doi:10.1142/S2251171717500076
+
 ## Contributors
 
  * Ben Barsdell

python/bifrost/DataType.py

@@ -49,6 +49,9 @@
 from bifrost.libbifrost import _bf
 import numpy as np
 
+from bifrost import telemetry
+telemetry.track_module()
+
 # Custom dtypes to represent additional complex types
 # Note: These can be constructed using tuples
 #   E.g., np.ndarray([(0x10,), (0x32,)], dtype=ci4) # Special case

python/bifrost/GPUArray.py

@@ -1,5 +1,5 @@
 
-# Copyright (c) 2016, The Bifrost Authors. All rights reserved.
+# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved.
 # Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -26,12 +26,13 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import ctypes
 import numpy as np
 from bifrost.memory import raw_malloc, raw_free, memset, memcpy, memcpy2D
-from bifrost.libbifrost import _bf, _check, _string2space
 from bifrost.array import _array2bifrost # This doesn't exist!
 
+from bifrost import telemetry
+telemetry.track_module()
+
 class GPUArray(object):
     def __init__(self, shape, dtype, buffer=None, offset=0, strides=None):
         itemsize = dtype().itemsize

python/bifrost/Space.py

@@ -27,6 +27,9 @@
 
 from bifrost.libbifrost import _bf
 
+from bifrost import telemetry
+telemetry.track_module()
+
 SPACEMAP_TO_STR = {_bf.BF_SPACE_AUTO:         'auto',
                    _bf.BF_SPACE_SYSTEM:       'system',
                    _bf.BF_SPACE_MAPPED:       'mapped',

python/bifrost/addon/leda/bandfiles.py

@@ -31,6 +31,9 @@
 sys.path.append('..')
 import make_header
 
+from bifrost import telemetry
+telemetry.track_module()
+
 disk_names = [ "ledastorage", "longterm", "offsite", "data" ]
 ledaovro_names = [ "ledaovro1", "ledaovro2", "ledaovro3", "ledaovro4", "ledaovro5", "ledaovro6", "ledaovro7", "ledaovro8", "ledaovro9", "ledaovro10", "ledaovro11","ledaovro12" ]
 data_names = [ "data1", "data2" ]
@@ -197,4 +200,3 @@ def has_freq(self, frequencies):
     for f in self.files:
       if f.freq in frequencies: has.append(f.freq)
     return has
-

python/bifrost/addon/leda/blocks.py

@@ -42,6 +42,8 @@
 import numpy as np
 from bifrost import block
 
+from bifrost import telemetry
+telemetry.track_module()
 
 # Read a collection of DADA files and form an array of time series data over
 # many frequencies.