#!/usr/bin/env python

from typing import Any, Dict, Iterable, List, Optional

from collections import Counter

import os

import os.path

import json

ROOT = ".mypy_cache/3.5"

JsonDict = Dict[str, Any]

class CacheData:

def __init__(self, filename: str, data_json: JsonDict, meta_json: JsonDict,

data_size: int, meta_size: int) -> None:

self.filename = filename

self.data = data_json

self.meta = meta_json

self.data_size = data_size

self.meta_size = meta_size

@property

def total_size(self):

return self.data_size + self.meta_size

def extract_classes(chunks: Iterable[CacheData]) -> Iterable[JsonDict]:

def extract(chunks: Iterable[JsonDict]) -> Iterable[JsonDict]:

for chunk in chunks:

if isinstance(chunk, dict):

yield chunk

yield from extract(chunk.values())

elif isinstance(chunk, list):

yield from extract(chunk)

yield from extract([chunk.data for chunk in chunks])

def load_json(data_path: str, meta_path: str) -> CacheData:

with open(data_path) as ds:

data_json = json.load(ds)

with open(meta_path) as ms:

meta_json = json.load(ms)

data_size = os.path.getsize(data_path)

meta_size = os.path.getsize(meta_path)

return CacheData(data_path.replace(".data.json", ".*.json"),

data_json, meta_json, data_size, meta_size)

def get_files(root: str) -> Iterable[CacheData]:

for (dirpath, dirnames, filenames) in os.walk(root):

for filename in filenames:

if filename.endswith(".data.json"):

meta_filename = filename.replace(".data.json", ".meta.json")

yield load_json(

os.path.join(dirpath, filename),

os.path.join(dirpath, meta_filename))

def pluck(name: str, chunks: Iterable[JsonDict]) -> Iterable[JsonDict]:

return (chunk for chunk in chunks if chunk['.class'] == name)

def report_counter(counter: Counter, amount: Optional[int] = None) -> None:

for name, count in counter.most_common(amount):

print(f' {count: <8} {name}')

print()

def report_most_common(chunks: List[JsonDict], amount: Optional[int] = None) -> None:

report_counter(Counter(str(chunk) for chunk in chunks), amount)

def compress(chunk: JsonDict) -> JsonDict:

cache = {} # type: Dict[int, JsonDict]

counter = 0

def helper(chunk: Any) -> Any:

nonlocal counter

if not isinstance(chunk, dict):

return chunk

if len(chunk) <= 2:

return chunk

id = hash(str(chunk))

if id in cache:

return cache[id]

else:

cache[id] = {'.id': counter}

chunk['.cache_id'] = counter

counter += 1

for name in sorted(chunk.keys()):

value = chunk[name]

if isinstance(value, list):

chunk[name] = [helper(child) for child in value]

elif isinstance(value, dict):

chunk[name] = helper(value)

return chunk

out = helper(chunk)

return out

def decompress(chunk: JsonDict) -> JsonDict:

cache = {} # type: Dict[int, JsonDict]

def helper(chunk: Any) -> Any:

if not isinstance(chunk, dict):

return chunk

if '.id' in chunk:

return cache[chunk['.id']]

counter = None

if '.cache_id' in chunk:

counter = chunk['.cache_id']

del chunk['.cache_id']

for name in sorted(chunk.keys()):

value = chunk[name]

if isinstance(value, list):

chunk[name] = [helper(child) for child in value]

elif isinstance(value, dict):

chunk[name] = helper(value)

if counter is not None:

cache[counter] = chunk

return chunk

return helper(chunk)

def main() -> None:

json_chunks = list(get_files(ROOT))

class_chunks = list(extract_classes(json_chunks))

total_size = sum(chunk.total_size for chunk in json_chunks)

print(f"Total cache size: {total_size / (1024 * 1024):.3f} megabytes")

print()

class_name_counter = Counter(chunk[".class"] for chunk in class_chunks)

print("Most commonly used classes:")

report_counter(class_name_counter)

print("Most common literal chunks:")

report_most_common(class_chunks, 15)

build = None

for chunk in json_chunks:

if 'build.*.json' in chunk.filename:

build = chunk

break

original = json.dumps(build.data, sort_keys=True)

print(f"Size of build.data.json, in kilobytes: {len(original) / 1024:.3f}")

build.data = compress(build.data)

compressed = json.dumps(build.data, sort_keys=True)

print(f"Size of compressed build.data.json, in kilobytes: {len(compressed) / 1024:.3f}")

build.data = decompress(build.data)

decompressed = json.dumps(build.data, sort_keys=True)

print(f"Size of decompressed build.data.json, in kilobytes: {len(decompressed) / 1024:.3f}")

print("Lossless conversion back", original == decompressed)

'''var_chunks = list(pluck("Var", class_chunks))

report_most_common(var_chunks, 20)

print()

#for var in var_chunks:

# if var['fullname'] == 'self' and not (isinstance(var['type'], dict) and var['type']['.class'] == 'AnyType'):

# print(var)

#argument_chunks = list(pluck("Argument", class_chunks))

symbol_table_node_chunks = list(pluck("SymbolTableNode", class_chunks))

report_most_common(symbol_table_node_chunks, 20)

print()

print("Most common")

report_most_common(class_chunks, 20)

print()'''

if __name__ == '__main__':

main()