def handler(event, context):
    import requests
    import datetime
    import boto3
    import json
    import pandas as pd
    import csv
    import io
    from io import StringIO

    def create_con():
        os = boto3.client('s3',
            aws_access_key_id = 'ВАШ КЛЮЧ',
            aws_secret_access_key = 'ВАШ СЕКРЕТНЫЙ КЛЮЧ',
            region_name = 'central1',
            endpoint_url = 'https://storage.yandexcloud.net'
            )
        return os
    def get_maxpage(API):
        adress = API
        todos = requests.get(adress)
        stat = todos.status_code
        tbody  = json.loads(todos.text)
        maxpage = tbody["pages"]
        return maxpage

    def getAPI(text, API, page):
        adress = API + '&page=' + page
        todos = requests.get(adress)
        stat = todos.status_code
        todos.encoding = 'utf-8'
        todos.text
        tbody = todos.json()
        t2body = json.dumps(tbody, ensure_ascii=False)
        df = pd.DataFrame(columns=['id', 'text', 'page'])
        for item in tbody["items"]:
            df = df.append({'id': item['id'], 'text': text, 'page': page}, ignore_index=True)
        return {
            'statusCode': 200,
            'body': df,
        }
    
    def get_list_req(os):
        obj = os.get_object(Bucket='ВАШ БАКЕТ',Key='ПУТЬ К ФАЙЛУ requests.csv')
        body = obj['Body'].read()
        df = pd.read_csv(io.BytesIO(body))
        return df

    os = create_con() #Подключение

    list_req = get_list_req(os) #Получаю датафрейм запросов
    df = pd.DataFrame(columns=['id', 'text', 'page'])
    for index, row in list_req.iterrows(): #Цикл по запросам, генерю файлы json
        maxpage = get_maxpage(row['API']) #получаю кол-во страниц
        int(maxpage)
        for i in range(0, maxpage):
            req = getAPI(row['text'], row['API'], str(i))
            df = df.append(req["body"], ignore_index=True)
    csv_buffer = StringIO()
    df.to_csv(csv_buffer)
    os.put_object(Bucket="ВАШ БАКЕТ", Body=csv_buffer.getvalue(), Key="Lakehouse/Config/" + "All_id_for_parse.csv", StorageClass='STANDARD')

def handler(event, context):
    import requests
    import datetime
    import boto3
    import json
    import pandas as pd
    import csv
    import io
    from io import StringIO

    def create_con():
        os = boto3.client('s3',
            aws_access_key_id = 'ВАШ КЛЮЧ',
            aws_secret_access_key = 'ВАШ СЕКРЕТНЫЙ КЛЮЧ',
            region_name = 'central1',
            endpoint_url = 'https://storage.yandexcloud.net'
            )
        return os

    def getAPI(text, id):
        adress = 'https://api.hh.ru/' + 'vacancies/' + str(id)
        todos = requests.get(adress)
        stat = todos.status_code
        todos.encoding = 'utf-8'
        todos.text
        tbody = todos.json()
        t2body = json.dumps(tbody, ensure_ascii=False)
        now = datetime.date.today()
        file_key = 'Lakehouse/Data/raw/' + now.strftime('%Y-%m-%d') + '/' + text + '/' + str(id) + '.json'
        os.put_object(Bucket='ВАШ БАКЕТ', Body=t2body, Key=file_key, StorageClass='STANDARD')
    
    def get_list_req(os):
        obj = os.get_object(Bucket='ВАШ БАКЕТ',Key='Lakehouse/Config/All_id_for_parse.csv')
        body = obj['Body'].read()
        df = pd.read_csv(io.BytesIO(body))
        return df

    os = create_con() #Подключение

    list_req = get_list_req(os) #Получаю датафрейм запросов
    df = pd.DataFrame(columns=['id', 'text', 'page'])
    for index, row in list_req.iterrows(): #Цикл по запросам, генерю файлы json
        req = getAPI(row['text'], row['id'])

def handler(event, context):
    import pandas as pd
    import requests
    from datetime import datetime, date
    import boto3
    import json
    import io
    from io import StringIO
    from io import BytesIO
    import pyarrow, fastparquet
    import urllib3
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
    mydate = str(date.today())
    data=[]
    columns=['id', 'name', 'salary', 'experience', 'schedule', 'employment', 'key_skills', 'employerName', 'accept_temporary',\
                                     'publishedAt', 'arhived', 'areaId', 'areaName', 'alternate_url', 'working_days', 'working_time_intervals', 'DateLoad', 'fileKey']
    df = pd.DataFrame(data, columns=columns)
                      
    df3 = df
    df_mart = df
    def create_con():
        os = boto3.client('s3',
            aws_access_key_id = 'ВАШ КЛЮЧ',
            aws_secret_access_key = 'ВАШ СЕКРЕТНЫЙ КЛЮЧ',
            region_name = 'central1',
            endpoint_url = 'https://storage.yandexcloud.net',
            verify=False
            )
        return os
    
    def get_all_s3_keys(os, bucket, prefix):
        """Get a list of all keys in an S3 bucket."""
        df = pd.DataFrame(columns=['file_key'])
        kwargs = {'Bucket': bucket, 'Prefix': prefix}
        while True:
            resp = os.list_objects_v2(**kwargs)
            for obj in resp['Contents']:
                df = pd.concat([pd.DataFrame([[obj['Key']]], columns=df.columns), df], ignore_index=True)
            try:
             kwargs['ContinuationToken'] = resp['NextContinuationToken']
            except KeyError:
                break   
        return df 

    def go_file(os, Key):
        obj = os.get_object(Bucket='ВАШ БАКЕТ',Key=Key) #Получаю файл джон
        todos = json.loads(obj['Body'].read())
        if todos["experience"] is not None:
            experience = todos["experience"]["id"]
        if todos["schedule"] is not None:
            schedule = todos["schedule"]["id"]
        if todos["employment"] is not None:
            employment = todos["employment"]["id"]
        try:
            employerName = todos["employer"]["name"]
        except:
            employerName = None
        try:
            areaId = todos["area"]["id"]
            areaName = todos["area"]["name"]
        except:
            areaId = None
            areaName = None
        row = [todos["id"], todos["name"], todos["salary"], experience, schedule, employment, todos["key_skills"], employerName, todos["accept_temporary"],\
                todos["published_at"],\
               todos["archived"], areaId, areaName, todos["alternate_url"], todos["working_days"], todos["working_time_intervals"], datetime.now(), Key]
        return pd.DataFrame([row], columns=columns)
    
    os = create_con() #Подключение
    df_list_file_key = get_all_s3_keys(os, 'ВАШ БАКЕТ', 'Lakehouse/Data/raw/' + mydate) #Получаю все ключи файлов
    for ind in df_list_file_key.index:
            df_res = go_file(os, df_list_file_key['file_key'][ind])
            df3 = pd.concat([df3, df_res])
    out_buffer = BytesIO()
    try:
        obj_mart = os.get_object(Bucket='ВАШ БАКЕТ',Key='Lakehouse/Data/proccessed/datamart') #Получаю файл parquet
        df_mart =  pd.read_parquet(io.BytesIO(obj_mart['Body'].read()))
    except:
        y = 1
    df_datamart = pd.concat([df_mart,df3]).drop_duplicates(['fileKey'],keep='last').sort_values('fileKey')
    df_datamart.astype(str).to_parquet(out_buffer, index=False)
    os.put_object(Bucket="ВАШ БАКЕТ", Body=out_buffer.getvalue(), Key='Lakehouse/Data/proccessed/datamart', StorageClass='STANDARD')
    return 'OK'

boto3
datetime
pandas
fsspec
requests
pyarrow
fastparquet
urllib3

"number","text","API"
"1","Azure","https://api.hh.ru/vacancies?text=Azure&search_field=name"
"2","data scientist","https://api.hh.ru/vacancies?text=data scientist&search_field=name"
"3","data engineer","https://api.hh.ru/vacancies?text=data engineer&search_field=name"