Pyspark export to delimited file

 

Pyspark export to delimited file

def myConcat(*cols):
    concat_columns = []
    for c in cols[:-1]:
        concat_columns.append(F.coalesce(c, F.lit("*")))
        concat_columns.append(F.lit("\t"))  
    concat_columns.append(F.coalesce(cols[-1], F.lit("*")))
    return F.concat(*concat_columns)

# combined column 에 모든 변수를 \t 로 concat 한 값 저장 
data_text = data.withColumn("combined", myConcat(*data.columns)).select("combined")
data_text.coalesce(1).write.format("text").option("header", "false").mode("overwrite").save(path)

출처 : https://stackoverflow.com/questions/17837871/how-to-copy-file-from-hdfs-to-the-local-file-system