https://wiki.korotkin.co.il/ 2026-04-09T23:50:14+00:00 https://wiki.korotkin.co.il/ https://wiki.korotkin.co.il/_media/wiki/logo.png text/html 2022-01-03T16:03:50+00:00 Anonymous (anonymous@undisclosed.example.com) https://wiki.korotkin.co.il/kb/bigdata/spark/import_json?rev=1641225830&do=diff Import JSON import json # import json df = sc.wholeTextFiles('/user/yehuda/development/raw_data/*.json').flatMap(lambda x: json.loads(x[1])).toDF() jsonRDD = sc.wholeTextFiles("/user/yehuda/development/raw_data/file-*.json").map(lambda x: json.loads(x[1])) namesJson = sqlContext.read.json(jsonRDD) namesJson.printSchema ## see https://stackoverflow.com/questions/46521235/hivecontext-object-has-no-attribute-jsonrdd-spark-2-1-1 import json from pyspark.sql.types import StructField, Struc… text/html 2022-01-03T16:03:50+00:00 Anonymous (anonymous@undisclosed.example.com) https://wiki.korotkin.co.il/kb/bigdata/spark/ml_save_and_load_model?rev=1641225830&do=diff ML Save and Load model Train & Save from pyspark.ml import Pipeline from pyspark.ml.classification import LogisticRegression from pyspark.ml.feature import HashingTF, Tokenizer from pyspark import SparkContext from pyspark.streaming import StreamingContext from pyspark.sql.session import SparkSession # Create a local StreamingContext with two working thread and batch interval of 1 second sc = SparkContext("local[2]", "NetworkWordCountML") ssc = StreamingContext(sc, 1) spark = SparkSession(sc… text/html 2022-01-03T16:03:50+00:00 Anonymous (anonymous@undisclosed.example.com) https://wiki.korotkin.co.il/kb/bigdata/spark/pyspark_and_hbase?rev=1641225830&do=diff Spark HBase import sys import json from pyspark import SparkContext from pyspark.streaming import StreamingContext def SaveRecord(rdd): host = 'sparkmaster.example.com' table = 'cats' keyConv = "org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter" valueConv = "org.apache.spark.examples.pythonconverters.StringListToPutConverter" conf = {"hbase.zookeeper.quorum": host, "hbase.mapred.outputtable": table, "mapreduce.o… text/html 2022-01-03T16:03:50+00:00 Anonymous (anonymous@undisclosed.example.com) https://wiki.korotkin.co.il/kb/bigdata/spark/spark_scala_and_maven_pom.xml?rev=1641225830&do=diff Spark Scala and Maven pom.xml <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.alefbt.bigdata.ml</groupId> <artifactId>mlserv</artifactId> <version>0.0.1-SNAPSHOT</version> <name>MLServe</name> <properties> <maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.target…