Skip to content
This repository was archived by the owner on Mar 10, 2025. It is now read-only.

Fix unusable method if DataFrame wanted instead of DataFrameStreaming #309

Open
piffall wants to merge 1 commit into
base: 2.4
Choose a base branch
from
Open

Fix unusable method if DataFrame wanted instead of DataFrameStreaming #309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ limitations under the License.
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>documentdb-bulkexecutor</artifactId>
<version>2.4.3-SNAPSHOT</version>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
Expand Down
10 changes: 5 additions & 5 deletions src/main/scala/com/microsoft/azure/cosmosdb/spark/schema/DataFrameReaderFunctions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,15 @@ private[spark] case class DataFrameReaderFunctions(@transient dfr: DataFrameRead
* @return DataFrame
*/
def cosmosDB[T <: Product : TypeTag](readConfig: Config): DataFrame =
createCosmosDBDataFrame(InferSchema.reflectSchema[T](), Some(readConfig), None)
createCosmosDBDataFrame(InferSchema.reflectSchema[T](), Option(readConfig), None)

/**
* Creates a [[DataFrame]] with the set schema
*
* @param schema the schema definition
* @return DataFrame
*/
def cosmosDB(schema: StructType): DataFrame = createCosmosDBDataFrame(Some(schema), None, None)
def cosmosDB(schema: StructType): DataFrame = createCosmosDBDataFrame(Option(schema), None, None)

/**
* Creates a [[DataFrame]] with the set schema
Expand All @@ -81,7 +81,7 @@ private[spark] case class DataFrameReaderFunctions(@transient dfr: DataFrameRead
* @param readConfig any custom read configuration
* @return DataFrame
*/
def cosmosDB(schema: StructType, readConfig: Config, sqlContext: SQLContext): DataFrame = createCosmosDBDataFrame(Some(schema), Some(readConfig), Some(sqlContext))
def cosmosDB(schema: StructType, readConfig: Config, sqlContext: SQLContext): DataFrame = createCosmosDBDataFrame(Option(schema), Option(readConfig), Option(sqlContext))

private def createDataFrame(schema: Option[StructType], readConfig: Option[Config], sqlContext: Option[SQLContext]): DataFrame = {
var cachingMode: CachingMode = CachingMode.NONE
Expand Down Expand Up @@ -135,12 +135,12 @@ private[spark] case class DataFrameReaderFunctions(@transient dfr: DataFrameRead
.-(CosmosDBConfig.RollingChangeFeed)
.-(CosmosDBConfig.CachingModeParam)
.+((CosmosDBConfig.CachingModeParam, CachingMode.CACHE.toString)))
val df = createDataFrame(schema, Some(dfConfig), sqlContext)
val df = createDataFrame(schema, Option(dfConfig), sqlContext)

val changeFeedConfig = Config(dfConfig.asOptions
.+((CosmosDBConfig.ReadChangeFeed, "true"))
.-(CosmosDBConfig.CachingModeParam))
val changeFeedDf = createDataFrame(schema, Some(changeFeedConfig), sqlContext)
val changeFeedDf = createDataFrame(schema, Option(changeFeedConfig), sqlContext)

df.union(changeFeedDf)
} else {
Expand Down