https://wiki.korotkin.co.il/ 2026-04-10T17:06:54+00:00 https://wiki.korotkin.co.il/ https://wiki.korotkin.co.il/_media/wiki/logo.png text/html 2022-01-03T16:03:50+00:00 Anonymous (anonymous@undisclosed.example.com) https://wiki.korotkin.co.il/kb/howto/create_and_save_simple_spark_ml_pipline?rev=1641225830&do=diff Create and save simple spark ml pipline # Import standard PySpark Transformers and packages from pyspark.ml.feature import VectorAssembler, StandardScaler, OneHotEncoder, StringIndexer from pyspark.ml import Pipeline, PipelineModel from pyspark.sql import Row # Create a test data frame l = [('Alice', 1), ('Bob', 2)] rdd = sc.parallelize(l) Person = Row('name', 'age') person = rdd.map(lambda r: Person(*r)) df2 = spark.createDataFrame(person) df2.collect() # Build a very simple pipeline using… text/html 2022-01-03T16:03:50+00:00 Anonymous (anonymous@undisclosed.example.com) https://wiki.korotkin.co.il/kb/howto/create_simple_dataframe_from_array?rev=1641225830&do=diff Create simple dataframe from array pr = [['Alice'], ['Bob']] pdf = sc.parallelize(pr).toDF(['name']) pdf.show()