# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # My Music 

# +
## Basic stuff
# %load_ext autoreload
# %autoreload
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
display(HTML("""<style>div.output_area{max-height:10000px;overflow:scroll;}</style>"""))

from mp3id import mp3ID
from musicBase import musicBase
from musicFinder import musicFinder
from musicPath import pathBasics
from timeUtils import clock, elapsed
from discogsBase import discogs
from multiArtist import multiArtist
from time import sleep
from pandas import DataFrame, Series
#from discogs import discogs

## Python Version
import sys
print("Python: {0}".format(sys.version))

import datetime as dt
start = dt.datetime.now()
print("Notebook Last Run Initiated: "+str(start))
# -

# # Music Analyzer (My Music)

# +
# %load_ext autoreload
# %autoreload

from searchUtils import findDirsPattern, findWalkExt
from fileUtils import getBaseFilename, getFileBasics
from fsUtils import setFile
from ioUtils import saveFile, getFile
from timeUtils import clock, elapsed
from musicPath import pathBasics


##############################################################################################################################
# My Music
##############################################################################################################################
class musicAnalyzer(musicBase):
    def __init__(self, debug=False):
        self.name = "myanalyzer"
        musicBase.__init__(self, debug=debug)
        
        self.pb      = pathBasics()
        self.flists  = self.getMusicListFiles()
        self.tagDBs  = self.getMusicTagDBs()
        self.pathDBs = self.getMusicPathDBs()


# +

def realName(x):
    if x is None:
        return [None,-1]
    
    lenx = len(x)
    if len(x) < 1:
        return [x,-1]

    if x[-1] != ")":
        return [x, None]
    

    if lenx >=5:
        if x[-3] == "(":
            try:
                num = int(x[-2:-1])
                val = x[:-3].strip()
                return [val, num]
            except:
                return [x, None]
            
    if lenx >= 6:
        if x[-4] == "(":
            try:
                num = int(x[-3:-1])
                val = x[:-4].strip()
                return [val, num]
            except:
                return [x, None]
            
    if lenx >= 7:
        if x[-4] == "(":
            try:
                num = int(x[-3:-1])
                val = x[:-4].strip()
                return [val, num]
            except:
                return [x, None]

    return [x, None]

# +
from pandas import Series
discdf = DataFrame(Series(df))
discdf.columns = ['Name']
tmp = DataFrame(discdf["Name"].apply(realName).tolist())
tmp.index = discdf.index
discdf['Artist'] = tmp[0]

discdf = discdf[~discdf['Name'].isna()]
print(discdf.shape)

artists = list(discdf["Artist"])

_ = clock("Last Run")

# +
# %load_ext autoreload
# %autoreload
mf = musicFinder(debug=False)
## Run This To Find Music Paths
mf.findMusic() #"Soundtrack", "Live", "Classical", "Jazz", "MixTape", "Compilation", "Non Fiction", "Todo", "Moves", "Not In Discogs"])
ma = musicAnalyzer(debug=False)

_ = clock("Last Run")

# +
start, cmt = clock("Getting My Artists/Albums")

artistData = {}
albumData = {}
for pathfile in ma.pathDBs:
#    if sum([x in pathfile for x in ["Soundtrack", "Live", "Classical", "Jazz", "MixTape", "Compilation", "Non Fiction", "Todo", "Moves", "Not In Discogs"]]) > 0:
#        continue
#    if sum([x in pathfile for x in ["Matched"]]) == 0:
#        continue
    pathdata = getFile(pathfile)
    for mp3,mp3data in pathdata.items():
        if artistData.get(mp3data['pbArtist']) is None:
            artistData[mp3data["pbArtist"]] = {}
        if artistData[mp3data["pbArtist"]].get(mp3data["pbAlbum"]) is None:
            artistData[mp3data["pbArtist"]][mp3data["pbAlbum"]] = []
        artistData[mp3data["pbArtist"]][mp3data["pbAlbum"]].append(mp3)

        if albumData.get(mp3data["pbAlbum"]) is None:
            albumData[mp3data["pbAlbum"]] = {}
        if albumData[mp3data["pbAlbum"]].get(mp3data["pbArtist"]) is None:
            albumData[mp3data["pbAlbum"]][mp3data["pbArtist"]] = []
        albumData[mp3data["pbAlbum"]][mp3data["pbArtist"]].append(mp3)
        
    print("Found {0} artists and {1} albums".format(len(artistData), len(albumData)))
    
    
saveFile(idata=artistData, ifile="/Volumes/Music/MusicDB/artistData.p")
saveFile(idata=albumData, ifile="/Volumes/Music/MusicDB/albumData.p")

elapsed(start, cmt)



_ = clock("Last Run")
# -

# # Find Artists

disc = discogs()
df = disc.getArtistIDToNameData()
tmp = Series(df).apply(realName).tolist()
artists = list(DataFrame(tmp)[0])

# +
from searchUtils import findNearest
from unicodedata import normalize
from pandas import DataFrame

def getMusicData(key, artist):
    retval = discdf[discdf[key] == artist]
    if retval.shape[0] > 0:
        return retval
    else:
        return None


saveArtistData = {"Odd": {}, "Match": {}, "Near": {}, "None": {}}

for artistName,artistAlbums in artistData.items():
    
    rename = None
    if False:
        rename = None
        if renameReplacements.get(artistName) is not None:
            artistName = renameReplacements[artistName]
            rename = True
        
    
    artistName = normalize('NFC', artistName)

    
    keys = ["Artist", "Name"]
    done = False
    for key in keys:
        mdata = getMusicData(key, artistName)
        if isinstance(mdata, DataFrame):
            if key == "Artist":
                #print('Match\t',mdata.shape,'\t',artistName)
                saveArtistData["Match"][artistName] = {"Match": artistName, "Rename": rename, "Discog": mdata, "Albums": artistAlbums}
            else:
                print('Odd\t',mdata.shape,'\t',artistName)
                saveArtistData["Odd"][artistName] = {"Match": artistName, "Rename": rename, "Discog": mdata, "Albums": artistAlbums}
            done = True
            break
    if done:
        continue

    cutoff=0.9
    if len(artistName) <= 6:
        cutoff = 0.8
    cutoffArtists = findNearest(item=artistName, ilist=artists, num=1, cutoff=cutoff)
    if len(cutoffArtists) > 0:
        for artist in cutoffArtists:
            mdata = getMusicData("Artist", artist)
            saveArtistData["Near"][artistName] = {"Match": artist, "Rename": rename, "Discog": mdata, "Albums": artistAlbums}
            print('Near','\t',mdata.shape,'\t',artistName,'\t\t====> Discog Match =',artist)
            done = True
    else:
        if False:
            cutoffArtists = findNearest(item=artistName, ilist=artists, num=1, cutoff=cutoff - 0.15)
            if len(cutoffArtists) > 0:
                for artist in cutoffArtists:
                    mdata = getMusicData("Artist", artist)
                    saveArtistData["Near"][artistName] = {"Match": artist, "Rename": rename, "Discog": mdata, "Albums": artistAlbums}
                    print('Near','\t',mdata.shape,'\t',artistName,'\t\t====> Discog Match =',artist)
                    done = True

        
    if done:
        continue
        
    print('\t','\t','\t',artistName)
    saveArtistData["None"][artistName] = {"Match": None, "Rename": rename, "Discog": None, "Albums": artistAlbums}
    continue
    
    
    if artistName == "<NAME> (2)":
        print(artistAlbums)
        1/0
    continue
    for albumName,albumVals in artistAlbums.items():
        print("\t",albumName,len(albumVals))    
    break
#albumData = {}
# -

# # Known Discog Renames

from pandas import read_excel
renames = read_excel('/Volumes/Music/Discog/db/DiscogRenames.xlsx')[["Unnamed: 1", "Unnamed: 2"]][1:]
renames.columns = ["My Name", "Discog Name"]
renames

cutoff=0.8
for discogName in renames["Discog Name"].values:
    cutoffArtists = findNearest(item=discogName, ilist=artists, num=1, cutoff=cutoff)
    print(discogName,'\t',cutoffArtists)

renames["Discog Name"].values

# # Full Matches

del saveArtistData["Match"]['11']

# +
toget = []
for artistName,aData in saveArtistData["Match"].items():
    rename = aData["Rename"]
    discog = aData["Discog"]
    albums = aData["Albums"]
    if discog is not None:
        matches = discog["Name"].values
        for val in matches:
            toget.append(val)
    else:
        continue
        
saveFile(idata=toget, ifile="/Volumes/Music/MusicDB/toget.p")

# +
toget2 = []
for artistName,aData in saveArtistData["Near"].items():
    rename = aData["Rename"]
    discog = aData["Discog"]
    albums = aData["Albums"]
    if discog is not None:
        matches = discog["Name"].values
        for val in matches:
            if val not in toget:
                toget2.append(val)
    else:
        continue
        

for artistName,aData in saveArtistData["Odd"].items():
    rename = aData["Rename"]
    discog = aData["Discog"]
    albums = aData["Albums"]
    if discog is not None:
        matches = discog["Name"].values
        for val in matches:
            if val not in toget:
                toget2.append(val)
    else:
        continue        
        
saveFile(idata=toget2, ifile="/Volumes/Music/MusicDB/toget2.p")

# +
toget3 = []
for i,(artistName,aData) in enumerate(saveArtistData["None"].items()):
    rename = aData["Rename"]
    discog = aData["Discog"]
    albums = aData["Albums"]

    matches = mulArts.getArtistNames(artistName)
    for name,val in matches.items():
        if val is None:
            continue
        if isinstance(val,list):
            for artistID in val:
                toget3.append(artistID)
        else:
            toget3.append(val)
        
toget3 = list(set(toget3))
saveFile(idata=toget3, ifile="/Volumes/Music/MusicDB/toget3.p")
# -

saveFile(idata=toget3, ifile="/Volumes/Music/MusicDB/toget3.p")

# # Mark Soundtracks

from searchUtils import findDirsPattern, findAll
from fsUtils import moveDir
from fileUtils import getDirBasics
for dirval in findDirsPattern('/Volumes/Music/iTunes Compilation/iTunes Media/Music', pattern="OST"):
    if dirval.endswith("OST"):
        dst = " ".join([dirval[:-3].strip(), "Soundtrack"])
        albumName = " ".join([getDirBasics(dirval)[-1][:-3].strip(), "Soundtrack"])
        for val in findAll(dirval):
            for albumFile in findAll(val):
                m = mp3ID(albumFile)
                m.setAlbum(albumName)
                
        moveDir(dirval, dst)

# # Near Matches

toExcel = []
for i,(artistName,aData) in enumerate(saveArtistData["Near"].items()):
    print("{0: <3}/{1: <3}".format(i,len(saveArtistData["Near"])), end=" \t")
    rename = aData["Rename"]
    discog = aData["Discog"]
    albums = aData["Albums"]
    if discog is not None:
        matches = discog["Name"].values
        print('{0: <50}'.format(artistName),matches)
    else:
        print('{0: <50}{1: <8}{2}'.format(artistName,'x',len(albums)))

# # Odd Names

for artistName,aData in saveArtistData["Odd"].items():
    rename = aData["Rename"]
    discog = aData["Discog"]
    albums = aData["Albums"]
    if discog is not None:
        matches = discog["Name"].values
        
        print('{0: <50}'.format(artistName),matches)
    else:
        print('{0: <50}{1: <8}{2}'.format(artistName,'x',len(albums)))
        

# # None Match

# +
## Abwaerts = Abwärts
## Move all ' vs ' into MixTape

## Known
#Beatsnblends
#Dr. <NAME>
# -

#artistData['<NAME> & FM']
albumData["Flo.Rida-Only.One.Flo.Pt.1-(Retail)-2010-NoFS)"]

#
# # Find Matches











# +
from searchUtils import findNearest

class multiArtist:
    def __init__(self, cutoff=0.9, discdata=None, exact=False):
        self.cutoff   = cutoff    
        self.discdata = discdata
        self.exact    = exact
        
        self.basicdelims = ["Duet With", "Presents", "Featuring"]
        self.delims = [",", "&", " And ", "+", "/", "With The", " with ", " With ", " y ", " Y ", " feat.",  " ft.", " Feat. ", " x ", " X "]
        self.discArtists = []
        if self.discdata is not None:
            self.discArtists = [x for x in discdata.keys() if x is not None]
        self.knownDelimArtists = {artist: True for artist in self.discArtists if self.nDelims(artist) > 0}

    
    def getDiscArtists(self):
        return self.discArtists
    
    def getKnownDelimArtists(self):
        return self.knownDelimArtists  
    
    def isKnownArtist(self, artist):
        if isinstance(self.discdata, dict):
            return self.discdata.get(artist) != None
        return False
    
    def nDelims(self, artist):
        return sum([x in artist for x in self.delims])
    
    
    def joinNames(self, delim, names):
        joinVal = delim
        if delim.startswith(" ") is False:
            joinVal = " {0}".format(joinVal)
        if delim.endswith(" ") is False:
            joinVal = "{0} ".format(joinVal)
        
        return joinVal.join(names)
    
    
    def getDelimData(self, val, debug=False):
        valdata = {d: [x.strip() for x in val.split(d)] for d in self.delims if d in val}
        if debug:
            print("\tDelimParsing ==> {0}".format(valdata))
        for k,v in valdata.items():
            if len(v) == 3:
                v.append(self.joinNames(delim=k, names=v[:2]))
                v.append(self.joinNames(delim=k, names=v[1:-1]))
        if debug:
            print("\tDelimParsing ==> {0}".format(valdata))

        return valdata
    
    
    def getBasicDelimData(self, val):
        valdata = {d: [x.strip() for x in val.split(d)] for d in self.basicdelims if d in val}
        return valdata

    def getNdelims(self, val):
        return len(val)

    def addArtist(self, allArtists, val, debug=False):
        #print("L={0}".format(len(allArtists)))
        if allArtists.get(val) is None:
            if debug:
                print("Adding {0}. Sum is {1}".format(val, len(allArtists)))
            allArtists[val] = True
    
    
    def cleanArtist(self, artist):
        artist = artist.replace("(", "")
        artist = artist.replace(")", "")
        return artist


    def newMethod(self, artist, debug=False):
        if debug is True:
            print("Parsing [{0}]".format(artist))
        artist = self.cleanArtist(artist)
        allArtists = {}
        d1delims = self.getBasicDelimData(artist)
        if len(d1delims) == 0:
            d1delims = self.getDelimData(artist)
        if debug:
            print('1','\t',artist,'===>',d1delims)

        knownArtists = set()
        if len(d1delims) == 0:
            self.addArtist(allArtists, artist, debug)
            knownArtists = set(allArtists.keys())
        elif self.isKnownArtist(artist):
            self.addArtist(allArtists, artist, debug)
            knownArtists = set(allArtists.keys())
            d1delims = {}


        ##############################################################################
        ## 1st Delimiter Split
        ##############################################################################
        for delim1, delimdata1 in d1delims.items():
            delimdata1 = list(set(delimdata1).difference(knownArtists))
            for artist1 in delimdata1:
                d2delims = self.getDelimData(artist1)
                if debug:
                    print('2','\t',artist1,'===>',d2delims)
                if self.getNdelims(d2delims) == 0:
                    self.addArtist(allArtists, artist1, debug)
                    knownArtists = set(allArtists.keys())
                    continue
                elif self.isKnownArtist(artist1):
                    self.addArtist(allArtists, artist1, debug)
                    knownArtists = set(allArtists.keys())
                    d2delims = {}

                    
                ##############################################################################
                ## 2nd Delimiter Split
                ##############################################################################
                for delim2, delimdata2 in d2delims.items():
                    delimdata2 = list(set(delimdata2).difference(knownArtists))
                    for artist2 in delimdata2:
                        d3delims = self.getDelimData(artist2)
                        if debug:
                            print('3','\t',artist2,'===>',d3delims)
                        if self.getNdelims(d3delims) == 0:
                            self.addArtist(allArtists, artist2)
                            knownArtists = set(allArtists.keys())
                            continue
                        elif self.isKnownArtist(artist2):
                            self.addArtist(allArtists, artist2)
                            knownArtists = set(allArtists.keys())
                            d3delims = {}

                            
                        ##############################################################################
                        ## 3rd Delimiter Split
                        ##############################################################################
                        for delim3, delimdata3 in d3delims.items():
                            delimdata3 = list(set(delimdata3).difference(knownArtists))
                            for artist3 in delimdata3:
                                d4delims = self.getDelimData(artist3)
                                if self.getNdelims(d4delims) == 0:
                                    self.addArtist(allArtists, artist3)
                                    knownArtists = set(allArtists.keys())
                                    continue
                                elif self.isKnownArtist(artist3):
                                    self.addArtist(allArtists, artist3)
                                    knownArtists = set(allArtists.keys())
                                    d4delims = {}
                                    

                                ##############################################################################
                                ## 4th Delimiter Split
                                ##############################################################################
                                for delim4, delimdata4 in d4delims.items():
                                    delimdata4 = list(set(delimdata4).difference(knownArtists))
                                    for artist4 in delimdata4:
                                        d5delims = self.getDelimData(artist4)
                                        if self.getNdelims(d5delims) == 0:
                                            self.addArtist(allArtists, artist4)
                                            knownArtists = set(allArtists.keys())
                                            continue
                                        elif self.isKnownArtist(artist4):
                                            self.addArtist(allArtists, artist4)
                                            knownArtists = set(allArtists.keys())
                                            d5delims = {}


                                        ##############################################################################
                                        ## 5th Delimiter Split
                                        ##############################################################################
                                        for delim5, delimdata5 in d5delims.items():
                                            delimdata5 = list(set(delimdata5).difference(knownArtists))
                                            for artist5 in delimdata5:
                                                d6delims = self.getDelimData(artist5)
                                                if self.getNdelims(d6delims) == 0:
                                                    self.addArtist(allArtists, artist5)
                                                    knownArtists = set(allArtists.keys())
                                                    continue
                                                elif self.isKnownArtist(artist5):
                                                    self.addArtist(allArtists, artist5)
                                                    knownArtists = set(allArtists.keys())
                                                    d6delims = {}

                                                    

        ##############################################################################
        ## Combine Results
        ##############################################################################                                                    
        results = {}
        if self.discdata is not None and len(self.discArtists) > 0:
            for name in knownArtists:
                retval = self.discdata.get(name)
                if self.exact is False:
                    if retval is None:
                        retval = findNearest(name, self.discArtists, 1, self.cutoff)
                        if len(retval) == 1:
                            retval = self.discdata.get(retval[0])
                        else:
                            retval = None
                        
                results[name] = retval
        else:
            results = {k: ['?'] for k,v in allArtists.items()}
                
        return results

    def getArtistNames(self, artist, debug=False):        
        return self.newMethod(artist, debug)
    
    
    
        if self.nDelims(artist) == 0:
            names = {artist: []}
            return names
        
        if self.discdata is not None and len(self.discArtists) > 0:
            retval = self.discdata.get(artist)
            if retval is not None:
                return {artist: retval}
            else:
                retval = findNearest(artist, self.discArtists, 1, self.cutoff)
                if len(retval) == 1:
                    return {artist: self.discdata.get(retval[0])}        
        
        names = {artist: None}
        names = self.splitArtist(names)
        names = self.unravelDict(names)
        names = self.unravelDict(names)
        return names


    def unravelDict(self, dvals):
        fvals = {}
        for k,v in dvals.items():
            if isinstance(v, dict):
                for k2,v2 in v.items():
                    if isinstance(v2, dict):
                        for k3,v3 in v.items():
                            if isinstance(v3, dict):
                                for k4,v4 in v.items():
                                    fvals[k4] = v4
                            else:
                                fvals[k3] = v3
                    else:
                        fvals[k2] = v2
            else:
                fvals[k] = v

        return fvals


    def splitArtistDelim(self, artist, delval):
        names = {}    
        if delval not in artist:
            return None

        for val in artist.split(delval):
            val = val.strip()
            
            if self.discdata is not None and len(self.discArtists) > 0:
                retval = self.discdata.get(val)
                if retval is not None:
                    names[val] = retval
                else:
                    retval = findNearest(val, self.discArtists, 1, self.cutoff)
                    if len(retval) == 0:
                        names[val] = None
                    else:
                        names[val] = retval
            else:
                names[val] = [-1]

        if len(names) == 0:
            return None

        if any(names.values()) is False:
            return None

        return names


    def splitArtist(self, result):
        delims = self.delims
            
        #print("Input: {0}".format(result))
        for name in result.keys():
            #print("  Name -->",name,"\tCurrent Value -->",result[name])
            if result[name] is None:
                for delim in delims:
                    #print("\tDelim: {0}  for {1}".format(delim, name))
                    result2 = self.splitArtistDelim(name,delval=delim)
                    #print("\tDelim Result: {0}".format(result2))


                    if result2 is not None:
                        result[name] = result2
                        #print("\tName:",name,'\tResult:',result2)
                        for name2 in result2.keys():
                            if result2[name2] is None:
                                for delim2 in delims:
                                    #print("\t\tDelim2: {0}  for {1}".format(delim2, name2))
                                    result3 = self.splitArtistDelim(name2,delval=delim2)
                                    #print("\t\tDelim Result: {0}".format(result3))


                                    if result3 is not None:
                                        #print("\t\tName:",name2,'\tResult:',result3)
                                        result2[name2] = result3

                                        ## Breaking from delim2 
                                        break

                        ## Breaking from delim
                        break

        return result
# -

#artistDB = disc.getArtistNameToIDData()
artistNameToID = {}
for artistID, artistName in discdf["Artist"].to_dict().items():
    if artistNameToID.get(artistName) is None:
        artistNameToID[artistName] = []
    artistNameToID[artistName].append(artistID)

for i,(artistName,aData) in enumerate(saveArtistData["None"].items()):
    rename = aData["Rename"]
    discog = aData["Discog"]
    albums = aData["Albums"]

    matches = mulArts.getArtistNames(artistName)
    if all(matches.values()):
        for albumName,albumValueData in albums.items():
            if len(albumValueData) == 1:
                print('{0: <50}'.format(artistName),matches)
                print("\t{0}\t{1}\n".format(len(albumValueData), albumName))

#discdf[discdf.index == '730612']
discdf[discdf.index == '135946']

for artistName,aData in saveArtistData["None"].items():
    rename = aData["Rename"]
    discog = aData["Discog"]
    albums = aData["Albums"]

    matches = mulArts.getArtistNames(artistName)
    if not all(matches.values()):
        print('{0: <50}'.format(artistName),matches)

# +
x = artistNames[4]
items = artistNames
my_str_as_bytes = str.encode(x, 'utf-8')
print(my_str_as_bytes,type(my_str_as_bytes)) # ensure it is byte representation


from unicodedata import normalize
keys = ['NFC', 'NFKC', 'NFD', 'NFKD']
for key in keys:
    print(x,'<---',key)    
    print(findNearest(item=x, ilist=items, num=1, cutoff=1.0))
    my_str = normalize(key, x)
    print(findNearest(item=my_str, ilist=items, num=1, cutoff=1.0))
    print(my_str,type(my_str))
    my_str_as_bytes = str.encode(my_str, 'utf-8')
    print(my_str_as_bytes,type(my_str_as_bytes)) # ensure it is byte representation
    my_decoded_str = my_str_as_bytes.decode()
    print(my_decoded_str,type(my_decoded_str)) # ensure it is string representation
    print(findNearest(item=my_decoded_str, ilist=items, num=1, cutoff=1.0))
    print("")
    
# -

# # Test

ifile = "/Volumes/Music/iTunes MixTape/iTunes Media/Music/24 Shots/01 Intro (Power Of The Dollar).mp3"
isFile(ifile)

# +
x = artistNames[4]
x = '<NAME>'


items = artistNames

from unicodedata import normalize
keys = ['NFC', 'NFKC', 'NFD', 'NFKD']
for key in keys:
    print(x,'<---',key)    
    print(findNearest(item=x, ilist=items, num=1, cutoff=1.0))
    my_str = normalize(key, x)
    print(findNearest(item=my_str, ilist=items, num=1, cutoff=1.0))
    print(my_str,type(my_str))
    my_str_as_bytes = str.encode(my_str, 'utf-8')
    print(my_str_as_bytes,type(my_str_as_bytes)) # ensure it is byte representation
    my_decoded_str = my_str_as_bytes.decode()
    print(my_decoded_str,type(my_decoded_str)) # ensure it is string representation
    print(findNearest(item=my_decoded_str, ilist=items, num=1, cutoff=1.0))
    print("")
    
# -

m = mp3ID(ifile)
m.getInfo()

pb  = pathBasics(debug=False)
pbc = pb.getPaths(ifile)
pbc.getDict()

# +
from mp3id import mp3ID
mp3Vals = {}
for artistName,aData in saveArtistData['Odd'].items():
    print(artistName)
    discog = aData['Discog']
    artist = None
    if isinstance(discog, DataFrame):
        if discog.shape[0] == 1:
            artist = discog['Artist'][0] #.values
            
    if artist is None:
        continue
        
    for album,albumFiles in aData['Albums'].items():
        print('  ',album)
        for albumFile in albumFiles:
            print('\t',albumFile)
            m = mp3ID(albumFile)
            mArtist = m.getArtist()[0]
            if mArtist == artistName:
                print("\t\tArtist: {0} <--> {1}".format(mArtist,artist))
                m.setArtist(artist)

            mAlbumArtist = m.getAlbumArtist()[0]
            if mAlbumArtist == artistName:
                print("\t\tAlbumArtist: {0} <--> {1}".format(mAlbumArtist,artist))
                m.setAlbumArtist(artist)
            #mp3Vals[albumFile] = mid.getInfo()
            
        


# -

mp3Vals

# # Multi Artists

artistDB = disc.getArtistIDToNameData()
artistNameToID = {v: k for k,v in artistDB.items()}
mulArts  = multiArtist(cutoff=0.9, discdata=artistNameToID, exact=False)



#len(artistDB)
len(artistNameToID)

renameReplacements





findNearest(item="<NAME>", ilist=artistDB.keys(), num=10, cutoff=0.7)

# +
x = "<NAME>"

from unicodedata import normalize
keys = ['NFC', 'NFKC', 'NFD', 'NFKD']
for key in keys:
    continue
    print(x,'<---',key)    
    print(findNearest(item=x, ilist=items, num=1, cutoff=1.0))
    my_str = normalize(key, x)
    print(findNearest(item=my_str, ilist=items, num=1, cutoff=1.0))
    print(my_str,type(my_str))
    my_str_as_bytes = str.encode(my_str, 'utf-8')
    print(my_str_as_bytes,type(my_str_as_bytes)) # ensure it is byte representation
    my_decoded_str = my_str_as_bytes.decode()
    print(my_decoded_str,type(my_decoded_str)) # ensure it is string representation
    print(findNearest(item=my_decoded_str, ilist=items, num=1, cutoff=1.0))
    print("")
    
# -

for artistName,artistData in saveArtistData.items():
    found = artistData[0]
    if found is False:
        retval = ma.getArtistNames(artistName)
        if len(retval) > 1:
            artistNames = [k for k,v in retval.items() if v is not None]
            print("{0: <50}{1}".format(artistName,artistNames))
        else:
            print("{0: <50}{1}".format(artistName,artistData[0]))

lvals = list(discdf["Artist"])
test  = "*NSYNC"
from searchUtils import findNearest
findNearest(item=test, ilist=lvals, num=1, cutoff=0.9)


def getMusicData(artist):
    retval = musicdf[musicdf["Artist"] == artist]
    if retval.shape[0] > 0:
        return retval
    else:
        return None



# +
artistIDtoName = disc.getArtistIDToNameData()
artistIDtoRefs  = disc.getArtistIDToRefData()
albumIDtoName = disc.getAlbumIDToNameData()
albumIDtoRef   = disc.getAlbumIDToRefData()

artistIDtoAlbumNames = disc.getArtistIDAlbumNames()
artistIDtoAlbumIDs = disc.getArtistIDAlbumIDs()

artistMetaData = disc.getAlbumArtistMetaData()

# +
colArtistIDtoName = disc.getCollectionIDToNameData()
colArtistIDtoRefs = disc.getCollectionIDToRefData()
colArtistReftoCnts = disc.getCollectionRefCountsData()

artistIDtoName = disc.getArtistIDToNameData()
artistIDtoRefs  = disc.getArtistIDToRefData()
albumIDtoName = disc.getAlbumIDToNameData()
albumIDtoRef   = disc.getAlbumIDToRefData()

artistIDtoAlbumNames = disc.getArtistIDAlbumNames()
artistIDtoAlbumIDs = disc.getArtistIDAlbumIDs()

artistMetaData = disc.getAlbumArtistMetaData()



def splitMetaData(x):
    retval = {}
    if isinstance(x, dict):
        for k,v in x.items():
            retval[k] = [z[0] for z in v.most_common(3)]
    else:
        retval = None
    return retval

from pandas import Series, DataFrame
sArtistToRef  = Series(artistIDtoRefs)
sArtistToName = Series(artistIDtoName)
sAlbumToRef   = Series(albumIDtoRef)
sAlbumToName  = Series(albumIDtoName)

sArtistToAlbums = Series(artistIDtoAlbumIDs)
sArtistToAlbumNames = Series(artistIDtoAlbumNames)
sArtistAlbums = Series([dict(zip(x, y)) for x,y in list(zip(sArtistToAlbums.values, sArtistToAlbumNames))], index=sArtistToAlbums.index)

sArtistMetaData = Series(artistMetaData)
sArtistMetaData = sArtistMetaData.apply(splitMetaData)


cols = ["Ref"]
discdf = DataFrame(sArtistToRef)
discdf.columns = cols
discdf = discdf.join(DataFrame(sArtistToName))
cols += ["Name"]
discdf.columns = cols
tmp = DataFrame(DataFrame(sArtistMetaData)[0].tolist())
tmp.index = sArtistMetaData.index
discdf = discdf.join(tmp)
cols += ["Extra Artists", "Genres", "Styles"]
discdf.columns = cols
discdf = discdf.join(DataFrame(sArtistAlbums))
cols += ["Albums"]
discdf.columns = cols




sColArtistToRef  = Series(colArtistIDtoRefs)
sColArtistToName = Series(colArtistIDtoName)
sColArtistRefToCnts = Series(colArtistReftoCnts)

cols = ["Ref"]
coldiscdf = DataFrame(sColArtistToRef)
coldiscdf.columns = cols
coldiscdf = coldiscdf.join(DataFrame(sColArtistToName))
cols += ["Name"]
coldiscdf.columns = cols

colrefdf = DataFrame(sColArtistRefToCnts)
colrefdf.columns = ["Counts"]
colrefdf.reset_index(inplace=True)
colrefdf.columns = ["Ref", "Counts"]
coldiscdf = coldiscdf.merge(colrefdf, on="Ref")
coldiscdf.index = DataFrame(sColArtistToRef).index

coldiscdf = coldiscdf.merge(discdf, on=["Ref", "Name"], how='left')
coldiscdf.index = DataFrame(sColArtistToRef).index
# -



# +
sColArtistToRef  = Series(colArtistIDtoRefs)
sColArtistToName = Series(colArtistIDtoName)
sColArtistRefToCnts = Series(colArtistReftoCnts)

cols = ["Ref"]
coldiscdf = DataFrame(sColArtistToRef)
coldiscdf.columns = cols
coldiscdf = coldiscdf.join(DataFrame(sColArtistToName))
cols += ["Name"]
coldiscdf.columns = cols

colrefdf = DataFrame(sColArtistRefToCnts)
colrefdf.columns = ["Counts"]
colrefdf.reset_index(inplace=True)
colrefdf.columns = ["Ref", "Counts"]
coldiscdf = coldiscdf.merge(colrefdf, on="Ref")
coldiscdf.index = DataFrame(sColArtistToRef).index

coldiscdf = coldiscdf.merge(discdf, on=["Ref", "Name"], how='left')
coldiscdf.index = DataFrame(sColArtistToRef).index
# -

coldiscdf.to_pickle("ArtistDiscog.p")

coldiscdf[coldiscdf["Name"] == "<NAME>"]





colrefdf

coldiscdf.shape

coldiscdf.merge(colrefdf, on="Ref", how="left")

# +

coldiscdf.merge(sColArtistRefToCnts, on="")
# -

discdf.shape

sArtistToAlbums     = Series(artistIDtoAlbumIDs)
sArtistToAlbumNames = Series(artistIDtoAlbumNames)

s = Series([dict(zip(x, y)) for x,y in list(zip(sArtistToAlbums.values, sArtistToAlbumNames))], index=sArtistToAlbums.index)

s

fixes = {}
iDir  = None
debug = False
for ifile,iPathData in jzpathdata.items():
    iTagData = jztagdata[ifile]
    
    pathArtist = iPathData['pbArtist']
    
    maPathArtists     = ma.getArtistNames(pathArtist)
    maTagArtists      = ma.getArtistNames(iTagData['Artist'])
    maTagAlbumArtists = ma.getArtistNames(iTagData['AlbumArtist'])

    if maTagArtists == maTagAlbumArtists:
        if maTagAlbumArtists == maPathArtists:
            continue
            print("Looks good")
        else:
            if debug:
                print("\nDifferent Artists:")
                print("\tPath:  {0}".format(maPathArtists.keys()))
                print("\tTag:   {0}".format(maTagArtists.keys()))
                print("\tTagAA: {0}".format(maTagAlbumArtists.keys()))
            if iDir is None:
                iDir = ifile.split(pathArtist)[0]                
                fixes[iDir] = {}
                print("  Adding {0}".format(iDir))
            if fixes[iDir].get(pathArtist) is None:
                fixes[iDir][pathArtist] = {}
                print("\tAdding {0}".format(pathArtist))
            iData = ifile.split(pathArtist)[1][1:]
            fixes[iDir][pathArtist][iData] = {"Error": "Path", "Path": maPathArtists, "Artist": maTagArtists, "AlbumArtist": maTagAlbumArtists}
            print("\t  Adding {0}".format(iData))
    else:
        if debug:
            print("\nUnknown Artist to Discogs:")
            print("\tPath:  {0}".format(maPathArtists))
            print("\tTag:   {0}".format(maTagArtists))
            print("\tTagAA: {0}".format(maTagAlbumArtists))


        if iDir is None:
            iDir = ifile.split(pathArtist)[0]                
            fixes[iDir] = {}
            print("  Adding {0}".format(iDir))
        if fixes[iDir].get(pathArtist) is None:
            fixes[iDir][pathArtist] = {}
            print("\tAdding {0}".format(pathArtist))
        iData = ifile.split(pathArtist)[1][1:]
        fixes[iDir][pathArtist][iData] = {"Error": "Tag", "Path": maPathArtists, "Artist": maTagArtists, "AlbumArtist": maTagAlbumArtists}
        print("\t  Adding {0}".format(iData))


saveFile(idata=fixes, ifile="fixes.yaml")



disc = discogs()
bsdata = searchDiscogs("Duke Ellington with the Ron Collier Orchestra Collages", requireUS=True, release=True)

from webUtils import getHTML
from artist import artist
#bsdata = getHTML(bsdata)

# +

h4s = bsdata.findAll("h4")
h5s = bsdata.findAll("h5")



for ih4,h4 in enumerate(h4s):
    print(h4)
    spans = h4.findAll("span")
    ref   = None
    if len(spans) == 0:
        ref = h4.find("a")
    else:
        ref = spans[0].find("a")

    if ref is None:
        continue

    try:
        href   = ref.attrs.get('href')
        name   = ref.text.strip()
    except:
        print("Could not get name/href from {0}".format(ref))
        continue


# +

ah5s
# -

artistData

# +
albumData  = [x for x in h4s if len(x.findAll('a')) > 0]
albumData  = [x.findAll('a') for x in albumData if not x.find('a').attrs['href'].startswith("/sell/")]

artistData = [x.findAll("a") for x in h5s if len(x.findAll('span')) > 0]
#artistData = [x.findAll("a") for x in artistData]
# -

                    baseURL = disc.discogURL
                    url = urllib.parse.urljoin(baseURL, href)
                    self.downloadAlbumURL(url, savename)
                

def getRefName(ref):
    try:
        href   = ref.attrs.get('href')
        name   = ref.text.strip()
    except:
        href   = None
        name   = None
        
    return href,name


artistData

alb = album()

# +
baseURL = disc.discogURL

for (albums,artists) in zip(albumData,artistData):
    for album in albums:
        albumRef,albumName = getRefName(album)
        print(albumRef,'\t',albumName)
        try:
            albumID  = str(int(albumRef.split("/")[-1]))
        except:
            print("Could not get album ID from {0}".format(albumRef))
            albumID  = None
            
        print(albumID)
        url = urllib.parse.urljoin(baseURL, albumRef)
                
                
    for artist in artists:
        artistRef,artistName = getRefName(artist)        
        print(artistRef,'\t',artistName)
        
    
# -

album



# +
from searchUtils import findDirsPattern, findWalkExt
from fileUtils import getBaseFilename, getFileBasics
from ioUtils import saveFile, getFile
from timeUtils import clock, elapsed
from musicPath import pathBasics


##############################################################################################################################
# My Music
##############################################################################################################################
class musicFileAnalyzer(musicBase):
    def __init__(self, debug=False):
        self.name = "myfileanalyzer"
        musicBase.__init__(self, debug=debug)
        
        self.pb      = pathBasics()
        self.pathDBs = self.getMusicPathDBs()
        self.tagDBs  = self.getMusicTagDBs()


# -

mfa = musicFileAnalyzer()

# +
from fileUtils import getDirname
from collections import Counter

artistPathData = {}
albumPathData  = {}

for cls in ma.getMusicClasses():
    pathData = ma.getMusicPathDB(cls)
    tagData  = ma.getMusicTagDB(cls)
    
    start, cmt = clock("  Checking {0} mp3s".format(len(pathData)))
    for mp3,pb in pathData.items():
        
        artist = pb["pbArtist"]
        if artist is not None:
            if artistPathData.get(artist) is None:
                artistPathData[artist] = {}
        else:
            if artistPathData.get(artist) is None:
                artistPathData[artist] = {}

            
        album = pb["pbAlbum"]
        if album is not None:
            if artistPathData[artist].get(album) is None:
                artistPathData[artist][album] = {}
        else:
            if artistPathData[artist].get(album) is None:
                artistPathData[artist][album] = {}

            
        cls = pb["pbClass"]
        if album is not None:
            if artistPathData[artist][album].get(cls) is None:
                artistPathData[artist][album][cls] = {}
        else:
            if artistPathData[artist][album].get(cls) is None:
                artistPathData[artist][album][cls] = {}

                
        dirval = getDirname(mp3)
        
        if artistPathData[artist][album][cls].get(dirval) is None:
            artistPathData[artist][album][cls][dirval] = []
        artistPathData[artist][album][cls][dirval].append([pb, tagData[mp3]])
    elapsed(start, cmt)
    print("")
    
    
savename = setFile(ma.getDBDir(), "ArtistAlbumTagData.p")
saveFile(ifile=savename, idata=artistPathData, debug=True)

# -

savename = setFile(ma.getDBDir(), "ArtistAlbumTagData.p")
artistPathData = getFile(savename)

nArt = 0
for artist,artistData in artistPathData.items():
    nArt += 1
    if nArt == 10:
        break
    for album,albumData in artistData.items():
        for cls,clsData in albumData.items():
            for dirval,dirvalData in clsData.items():
                
                dval = dirval.replace(ma.getMusicDir(), "")
                
                nFiles = len(dirvalData)
                partist = artist
                if len(artist) >= 40:
                    partist = "{0}...".format(artist[:36])
                palbum = album
                if len(album) >= 40:
                    palbum = "{0}...".format(album[:36])

                print("{0: <40}{1: <40}{2: <20}{3: <5}{4}".format(partist,palbum,cls,nFiles,dirval))

artistPathData

for ifile in ma.flists:
    tagfile  = ifile.replace(".p", "-Tags.p")
    pathfile = ifile.replace(".p", "-Paths.p")

    fileData = getFile(ifile)
    tagData  = getFile(tagfile)
    pathData = getFile(pathfile)

from searchUtils import findExt
files = findExt(basedir=ma.getDBDir(), ext="*.p")
files = [x for x in files if sum([y in x for y in ["Tags.p", "Paths.p"]]) == 0]
files



import os, sys
def splitall(path):
    allparts = []
    while 1:
        parts = os.path.split(path)
        if parts[0] == path:  # sentinel for absolute paths
            allparts.insert(0, parts[0])
            break
        elif parts[1] == path: # sentinel for relative paths
            allparts.insert(0, parts[1])
            break
        else:
            path = parts[0]
            allparts.insert(0, parts[1])
    return allparts


os.path.split("24 Shots/01 Intro (Power Of The Dollar)/dfd")

idir

files[-2]



ifile



filename

dirval



artist









# %load_ext autoreload
# %autoreload
disc = discogs()

# +
import argparse as ap
from os import getcwd
from mp3id import mp3ID
mid = mp3ID(ifile)
minfo = mid.getInfo()
from glob import glob
from fsUtils import isFile
from fileUtils import getExt
from searchUtils import findWalkExt, findWalkPattern
from os import listdir, walk

class mp3Tagger:
    def __init__(self, disc=None):
        self.name = "mp3tagger"
        self.cwd  = getcwd()
        
        self.infos = {}
        self.ma       = multiArtist(cutoff=0.9, discdata=None, exact=False)

        
        if disc is not None:
            self.setDiscogs(disc)
        
        self.exts = mp3ID().mp3exts
        

    def setDiscogs(self, disc):
        artistDB = disc.getArtistVariationNameToIDsData()
        self.setArtistsDB(artistDB=artistDB)
        
    def setArtistsDB(self, artistDB, cutoff=0.9):
        self.ma       = multiArtist(cutoff=0.9, discdata=artistDB, exact=False)
        
        
    def getFiles(self, dirval):
        self.infos = {}
        files = findWalkPattern(basedir=dirval, pattern=".")
        files = [x for x in files if getExt(x) in self.exts]
        for i,ifile in enumerate(files):
            if self.infos.get(ifile) is None:
                mid = mp3ID(ifile)
                minfo = mid.getInfo()
                self.infos[ifile] = minfo
                
        
    def get(self):
        return self.infos
        
    def getInfo(self, dirval=None):
        if dirval is not None:
            self.getFiles(dirval)
            
        print("{0: <3}{1: <4}{2: <4}{3: <50}{4: <50}{5: <40}{6: <40}".format("#", "Trk", "Dsc", "Title", "Album", "Artist", "Album Artist"))

        i = 0
        for ifile,minfo in self.infos.items():
            print("{0: <3}".format(i+1), end="")

            trkno = minfo.get('TrackNo')
            if trkno is None:
                trkno = -1
            elif isinstance(trkno, list):
                if len(trkno) == 1:
                    trkno = trkno[0]
                else:
                    trkno = str(trkno)
            print("{0: <4}".format(trkno), end="")

            discno = minfo.get('DiscNo')
            if discno is None:
                discno = -1
            elif isinstance(discno, list):
                if len(discno) == 1:
                    discno = discno[0]
                else:
                    discno = str(discno)                
            print("{0: <4}".format(discno), end="")

            title = minfo.get('Title')
            if title is None:
                title = "?"
            elif isinstance(title, list):
                if len(title) == 1:
                    title = title[0]
                else:
                    title = str(title)
            print("{0: <50}".format(title), end="")

            album = minfo.get('Album')
            if album is None:
                album = "?"
            elif isinstance(album, list):
                if len(album) == 1:
                    album = album[0]
                else:
                    album = str(album)
            print("{0: <50}".format(album), end="")

            artist = minfo.get('Artist')
            if artist is None:
                artist = "?"
            elif isinstance(artist, list):
                if len(artist) == 1:
                    artist = artist[0]
                else:
                    artist = str(artist)
            print("{0: <40}".format(artist), end="")

            albartist = minfo.get('AlbumArtist')
            if albartist is None:
                albartist = "?"
            elif isinstance(albartist, list):
                if len(albartist) == 1:
                    albartist = albartist[0]
                else:
                    albartist = str(albartist)
            print("{0: <40}".format(albartist), end="")
            print("")
            
            i += 1
            
    

    def getAlbumArtists(self):
        return [x.get("AlbumArtist") for ifile,x in self.infos.items()]
# -

mt = mp3Tagger(disc=disc)

mt.getInfo(dirval="/Volumes/Music/iTunes Jazz/iTunes Media/Music/<NAME> & Qu<NAME>")

# +
from ioUtils import saveFile
retvals = {}
for ifile,minfo in mt.get().items():
    retvals[ifile] = {}

    for key in ["Artist", "AlbumArtist"]:
        val = minfo.get(key)
        if isinstance(val, list):
            if len(val) == 1:
                retval = mt.ma.getArtistNames(val[0])
                if len(retval) > 1:
                    artistNames = [k for k,v in retval.items() if v is not None]
                    retvals[ifile][key] = artistNames
                    
saveFile(idata=retvals, ifile="tmp.yaml")
# -

retvals

tmp=[['<NAME> & Qu<NAME>'],
 ['<NAME> & Qu<NAME>'],
 ['<NAME> & Qu<NAME>'],
 ['<NAME> & Qu<NAME>'],
 ['<NAME> & Qu<NAME>'],
 ['<NAME> & Quincy Jones'],
 ['<NAME> & Quincy Jones'],
 ['<NAME> & Quincy Jones'],
 ['<NAME> & Quincy Jones'],
 ['<NAME> & Quincy Jones'],
 ['<NAME> & Quincy Jones'],
 ['<NAME> & Quincy Jones'],
 ['<NAME> & Quincy Jones'],
 ['<NAME> & Qu<NAME>'],
 ['<NAME> & Quincy Jones'],
 ['<NAME> & Quincy Jones']]



m.getDBDir



# +
idir="/Volumes/Music/iTunes Jazz/iTunes Media/Music/<NAME> & <NAME>


from random import shuffle
print("There are {0} chart artists".format(len(chartArtists)))
shuffle(chartArtists)
for i,artist in enumerate(chartArtists):
    if prevs.get(artist) is not None:
        continue
    retval = ma.getArtistNames(artist)

StarcoderdataJupyterScriptsDedupFiltered