# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 [dev-base]
#     language: python
#     name: anaconda-project-dev-base-python3
# ---

import os
from hdfs.ext.kerberos import KerberosClient
import getpass
import uuid
import time

_NAMENODE_PORT = 50070


def get_client(namenode_url: str, namenode_port: int = None):
    port = namenode_port = _NAMENODE_PORT
    return KerberosClient('%s:%s' % (namenode_url, port))



# +
def upload(local_file: str,
           namenode_url: str,
           namenode_port: int = None,
           hdfs_dir: str = None,):
    """
    Parameters
    ----------
    local_file: The local file to upload
    namenode_url: The url of the namenode
    namenode_port: (optional) The port that the namenode service is running on. Defaults to 
        value of module-level variable _NAMENODE_PORT.
    hdfs_dir: (optional) The folder (or full path) to upload `local_file` to. If none is provided,
        then the file will be uploaded to the following directory: 
        /tmp/livy_{current user name}_{First six characters of uuid4}_{current time in seconds}'
        
    Returns
    -------
    str: The directory where `local_file` was uploaded so you can use it to upload the rest of your
        files needed for your Spark job
    """
    port = namenode_port or _NAMENODE_PORT
    
    client = get_client(namenode_url, port)
    
    if hdfs_dir is None:
        hdfs_dir = f'/tmp/livy_{getpass.getuser()}_{str(uuid.uuid4())[:6]}_{round(time.time())}/'
        
#     print(hdfs_dir)
    client.makedirs(hdfs_dir)
    
    resp = client.upload(hdfs_dir, local_file, overwrite=True)
#     print(resp)
    
    return hdfs_dir
    
    
    

# -

# ---
#
# Do some QA on the hdfs client functions

namenode_url = 

namenode_url = 'http://ip-172-31-20-241.ec2.internal'

upload('../QA/pi.py', namenode_url)

client.delete('/tmp/livy_ec2-user_dfe804_1545062025/', recursive=True)

client = get_client(namenode_url)

client.list('/tmp/livy_ec2-user_dfe804_1545062025/')

# +
# client._get_home_directory??
# -

client.upload?a

import sys
sys.path.insert(0, '../')

from livy_submit import hdfs_api

namenode_url = 'http://ip-172-31-20-241.ec2.internal:50070'

local_file = '../test/data/pi_runner.zip'

hdfs_dirname = '/user/edill/livy-submit-files/19d579_1545492416'

hdfs_api.upload(local_file, namenode_url, hdfs_dirname)



StarcoderdataJupyterScriptsDedupFiltered