pyspark基础函数

# -*- coding: utf-8 -*-
"""
Created on Sun Jul  3 15:38:33 2022

@author: 
"""

import json
from pyspark import StorageLevel
from pyspark.sql import Window
from pyspark.sql import SparkSession
from pyspark.sql import function as F
from pyspark.sql.types import StringType, ArrayType, MapType
from pyspark.sql.column import Column, _to_seq, _to_java_column


class AoiProfile(object):
    
    def __init__(self, city_code):

        self.city_code = city_code
        self.storage_level = StorageLevel.MEMORY_AND_DISK
        self.path_csv = "path/to/abc.csv"

        self.spark = SparkSession.\
            builder.\
            appName("aoi_profile").\
            config("spark.some.config.option", "some-value").\
            enableHiveSupport().\
            getOrCreate()
        self.spark.sparkContext.setLogLevel("ERROR")
        self.udf_get_dict = F.udf(self.get_d