Merge pull request #509 from s22s/docs/fix-build

Updates for clean CI build

Author: vpipkt

.circleci/config.yml

@@ -136,7 +136,6 @@ jobs:
       - run:
           name: "Create PyRasterFrames package"
           command: |-
-            python -m pip install --user pyspark==2.4.5
             sbt -v -batch pyrasterframes/package
 
       - run:

core/src/test/scala/org/locationtech/rasterframes/TestData.scala

@@ -141,14 +141,16 @@ trait TestData {
     rf.toTileLayerRDD(rf.tileColumns.head).left.get
   }
 
-  private val baseCOG = "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_%s.TIF"
-  lazy val remoteCOGSingleband1: URI = URI.create(baseCOG.format("B1"))
-  lazy val remoteCOGSingleband2: URI = URI.create(baseCOG.format("B2"))
+  // Check the URL exists as of 2020-09-30; strictly these are not COGs because they do not have internal overviews
+  private def remoteCOGSingleBand(b: Int) = URI.create(s"https://landsat-pds.s3.us-west-2.amazonaws.com/c1/L8/017/029/LC08_L1TP_017029_20200422_20200509_01_T1/LC08_L1TP_017029_20200422_20200509_01_T1_B${b}.TIF")
+  lazy val remoteCOGSingleband1: URI = remoteCOGSingleBand(1)
+  lazy val remoteCOGSingleband2: URI = remoteCOGSingleBand(2)
 
-  lazy val remoteCOGMultiband: URI =  URI.create("https://s3-us-west-2.amazonaws.com/radiant-nasa-iserv/2014/02/14/IP0201402141023382027S03100E/IP0201402141023382027S03100E-COG.tif")
+  // a public 4 band COG TIF
+  lazy val remoteCOGMultiband: URI = URI.create("https://s22s-rasterframes-integration-tests.s3.amazonaws.com/m_4411708_ne_11_1_20141005.cog.tif")
 
   lazy val remoteMODIS: URI = URI.create("https://modis-pds.s3.amazonaws.com/MCD43A4.006/31/11/2017158/MCD43A4.A2017158.h31v11.006.2017171203421_B01.TIF")
-  lazy val remoteL8: URI = URI.create("https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/017/033/LC08_L1TP_017033_20181010_20181030_01_T1/LC08_L1TP_017033_20181010_20181030_01_T1_B4.TIF")
+  lazy val remoteL8: URI = URI.create("https://landsat-pds.s3.amazonaws.com/c1/L8/017/033/LC08_L1TP_017033_20181010_20181030_01_T1/LC08_L1TP_017033_20181010_20181030_01_T1_B4.TIF")
   lazy val remoteHttpMrfPath: URI = URI.create("https://s3.amazonaws.com/s22s-rasterframes-integration-tests/m_3607526_sw_18_1_20160708.mrf")
   lazy val remoteS3MrfPath: URI = URI.create("s3://naip-analytic/va/2016/100cm/rgbir/37077/m_3707764_sw_18_1_20160708.mrf")
 

core/src/test/scala/org/locationtech/rasterframes/ref/RasterSourceSpec.scala

@@ -102,7 +102,7 @@ class RasterSourceSpec extends TestEnvironment with TestData {
     }
     it("should read sub-tile") {
       withClue("remoteCOGSingleband") {
-        val src = RFRasterSource(remoteCOGSingleband1)
+        val src = RFRasterSource(remoteMODIS)
         val raster = src.read(sub(src.extent))
         assert(raster.size > 0 && raster.size < src.size)
       }

docs/src/main/paradox/release-notes.md

@@ -4,6 +4,7 @@
 
 ### 0.9.1
 
+* Upgraded to Spark 2.4.7
 * Added `method_name` parameter to the `rf_resample` method.
    * __BREAKING__: In SQL, the function `rf_resample` now takes 3 arguments. You can use `rf_resample_nearest` with two arguments or refactor to `rf_resample(t, v, "nearest")`.
 * Added resample method parameter to SQL and Python APIs. @ref:[See updated docs](raster-join.md).

project/RFDependenciesPlugin.scala

@@ -73,7 +73,7 @@ object RFDependenciesPlugin extends AutoPlugin {
     },
     // dependencyOverrides += "com.azavea.gdal" % "gdal-warp-bindings" % "33.f746890",
     // NB: Make sure to update the Spark version in pyrasterframes/python/setup.py
-    rfSparkVersion := "2.4.5",
+    rfSparkVersion := "2.4.7",
     rfGeoTrellisVersion := "3.3.0",
     rfGeoMesaVersion := "2.2.1"
   )

pyrasterframes/src/main/python/docs/time-series.pymd

@@ -41,8 +41,6 @@ def simplify(g, tol):
 park_vector = park_vector.withColumn('geo_simp', simplify('geometry', lit(0.001))) \
                          .select('geo_simp') \
                          .hint('broadcast')
-
-
 ```
 
 ## Catalog Read
@@ -56,7 +54,7 @@ park_cat = cat \
                     (cat.acquisition_date > lit('2018-02-19')) &
                     (cat.acquisition_date < lit('2018-07-01'))            
                     ) \
-            .crossJoin(park_vector.filter('OBJECTID == 380')) #only coyahuga
+            .crossJoin(park_vector.filter('UNIT_CODE == "CUVA"')) #only coyahuga
 
 ```
 

pyrasterframes/src/main/python/requirements.txt

@@ -1,5 +1,5 @@
 ipython==6.2.1
-pyspark==2.4.5
+pyspark==2.4.7
 gdal==2.4.4
 numpy>=1.17.3,<2.0
 pandas>=0.25.3,<1.0

pyrasterframes/src/main/python/setup.py

@@ -144,11 +144,13 @@ def dest_file(self, src_file):
 ipython = 'ipython==6.2.1'
 jupyter_client = 'jupyter-client<6.0'  # v6 breaks pweave
 matplotlib = 'matplotlib'
+nbclient = 'nbclient==0.1.0'  # compatible with our pweave => jupyter_client restrictions
+nbconvert = 'nbconvert==5.5.0'
 numpy = 'numpy>=1.17.3,<2.0'
 pandas = 'pandas>=0.25.3,<1.0'
 pweave = 'pweave==0.30.3'
 pypandoc = 'pypandoc'
-pyspark = 'pyspark==2.4.5'
+pyspark = 'pyspark==2.4.7'
 pytest = 'pytest>=4.0.0,<5.0.0'
 pytest_runner = 'pytest-runner'
 pytz = 'pytz'
@@ -196,9 +198,10 @@ def dest_file(self, src_file):
         pytest_runner,
         setuptools,
         ipython,
-        ipykernel,
         pweave,
         jupyter_client,
+        nbclient,
+        nbconvert,
         fiona,
         rasterio,
         folium,