From 000a9ae6746f455557a897ab16bd5879ea0b0826 Mon Sep 17 00:00:00 2001 From: Federico-Varela Date: Thu, 6 Jun 2024 12:46:14 -0300 Subject: [PATCH] translating docs, adding examples --- tools/analytics/README.md | 103 +++++++++++++++++++++++++++ tools/analytics/requirements.txt | 62 ++++++++++++++++ tools/analytics/src/config.py | 37 ++++++++++ tools/analytics/src/ec2benchmarks.py | 34 +++++++++ tools/analytics/src/huggingface.py | 86 ++++++++++++++++++++++ tools/analytics/src/main.py | 78 ++++++++++++++++++++ tools/analytics/src/postprocess.py | 32 +++++++++ tools/analytics/src/preprocessing.py | 80 +++++++++++++++++++++ tools/analytics/src/render.py | 98 +++++++++++++++++++++++++ tools/analytics/templates/graph.html | 35 +++++++++ tools/analytics/templates/table.html | 97 +++++++++++++++++++++++++ 11 files changed, 742 insertions(+) create mode 100644 tools/analytics/README.md create mode 100644 tools/analytics/requirements.txt create mode 100644 tools/analytics/src/config.py create mode 100644 tools/analytics/src/ec2benchmarks.py create mode 100644 tools/analytics/src/huggingface.py create mode 100644 tools/analytics/src/main.py create mode 100644 tools/analytics/src/postprocess.py create mode 100644 tools/analytics/src/preprocessing.py create mode 100644 tools/analytics/src/render.py create mode 100644 tools/analytics/templates/graph.html create mode 100644 tools/analytics/templates/table.html diff --git a/tools/analytics/README.md b/tools/analytics/README.md new file mode 100644 index 0000000..7c4f742 --- /dev/null +++ b/tools/analytics/README.md @@ -0,0 +1,103 @@ +# LLM performance analysis + +## How to run +First, we preprocess the input to convert every file into TSV (tab separated values) format +Each input file type requires its own conversion function, the already supported formats are listed in `src/config.py` +``` +rm -r +python3 src/main.py --preprocess +``` + +This command generates a directory with all preprocessed files, named after the original directory with `_preprocessed` appended to it +Now we can run: +``` +python3 src/main.py _preprocessed +``` + +This will generate `index.html` and the visualizations under `` + +Some options for data and image format are available in `src/config.py` + + +## Example run +Raw data directory: +``` +examples +└── benchmarks + ├── '2024-04 Llama2-70b p4d.24xlarge' + │ ├── CTranslate2.csv + │ ├── CTranslate2.free + │ ├── CTranslate2.mpstat + │ └── CTranslate2.nvidia-smi + └── '2024-04 OpenAI' + ├── 'OpenAI gpt-3.5-turbo.csv' + ├── 'OpenAI gpt-3.5-turbo.free' + ├── 'OpenAI gpt-3.5-turbo.mpstat' + ├── 'OpenAI gpt-3.5-turbo.nvidia-smi' + ├── 'OpenAI gpt-4-turbo-preview.csv' + ├── 'OpenAI gpt-4-turbo-preview.free' + ├── 'OpenAI gpt-4-turbo-preview.mpstat' + └── 'OpenAI gpt-4-turbo-preview.nvidia-smi' +``` + +Running `python3 src/main.py --preprocess examples/benchmarks` creates: +``` +examples_preprocessed +└── benchmarks + ├── '2024-04 Llama2-70b p4d.24xlarge' + │ ├── CTranslate2.csv + │ ├── CTranslate2.free + │ ├── CTranslate2.mpstat + │ └── CTranslate2.nvidia-smi + └── '2024-04 OpenAI' + ├── 'OpenAI gpt-3.5-turbo.csv' + ├── 'OpenAI gpt-3.5-turbo.free' + ├── 'OpenAI gpt-3.5-turbo.mpstat' + ├── 'OpenAI gpt-3.5-turbo.nvidia-smi' + ├── 'OpenAI gpt-4-turbo-preview.csv' + ├── 'OpenAI gpt-4-turbo-preview.free' + ├── 'OpenAI gpt-4-turbo-preview.mpstat' + └── 'OpenAI gpt-4-turbo-preview.nvidia-smi' +``` +Each file is replicated with the same path and data but with a TSV format +Then, run `python3 src/main.py examples_preprocessed/benchmarks tmp` which creates: +``` +tmp +├── graph.png +├── index.html +├── table_csv.png +├── table_free.png +├── table_mpstat.png +└── table_nvidia-smi.png +``` + +## File preprocessing +Example .free file before preprocessing: +``` + total used free shared buff/cache available +Mem: 1148221 7438 881686 9 264579 1140783 +Swap: 0 0 0 + + total used free shared buff/cache available +Mem: 1148221 8174 880949 9 264582 1140046 +Swap: 0 0 0 + + total used free shared buff/cache available +Mem: 1148221 8434 880687 9 264584 1139786 +Swap: 0 0 0 +... +``` + +After preprocessing: +``` +type total used free shared buff/cache available +Mem: 1148221 7438 881686 9 264579 1140783 +Swap: 0 0 0 + +Mem: 1148221 8174 880949 9 264582 1140046 +Swap: 0 0 0 + +Mem: 1148221 8434 880687 9 264584 1139786 +Swap: 0 0 0 +... +``` diff --git a/tools/analytics/requirements.txt b/tools/analytics/requirements.txt new file mode 100644 index 0000000..341175f --- /dev/null +++ b/tools/analytics/requirements.txt @@ -0,0 +1,62 @@ +aiohttp==3.9.3 +aiosignal==1.3.1 +attrs==23.2.0 +beautifulsoup4==4.12.3 +bleach==6.1.0 +certifi==2024.2.2 +charset-normalizer==3.3.2 +contourpy==1.2.0 +cssutils==2.9.0 +cycler==0.12.1 +dataframe-image==0.2.3 +defusedxml==0.7.1 +fastjsonschema==2.19.1 +fonttools==4.48.1 +frozenlist==1.4.1 +html2image==2.0.4.3 +idna==3.6 +Jinja2==3.1.3 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 +jupyter_client==8.6.0 +jupyter_core==5.7.1 +jupyterlab_pygments==0.3.0 +kiwisolver==1.4.5 +lxml==5.1.0 +MarkupSafe==2.1.5 +matplotlib==3.8.2 +mistune==3.0.2 +multidict==6.0.5 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +numpy==1.26.4 +packaging==23.2 +pandas==2.2.0 +pandocfilters==1.5.1 +pillow==10.2.0 +platformdirs==4.2.0 +pyarrow==15.0.0 +Pygments==2.17.2 +pyparsing==3.1.1 +PyQt5==5.15.10 +PyQt5-Qt5==5.15.2 +PyQt5-sip==12.13.0 +python-dateutil==2.8.2 +pytz==2024.1 +pyzmq==25.1.2 +referencing==0.33.0 +requests==2.31.0 +rpds-py==0.18.0 +seaborn==0.13.2 +six==1.16.0 +soupsieve==2.5 +tabulate==0.9.0 +tinycss2==1.2.1 +tornado==6.4 +traitlets==5.14.1 +tzdata==2023.4 +urllib3==2.2.0 +webencodings==0.5.1 +websocket-client==1.7.0 +yarl==1.9.4 diff --git a/tools/analytics/src/config.py b/tools/analytics/src/config.py new file mode 100644 index 0000000..9ae0bc8 --- /dev/null +++ b/tools/analytics/src/config.py @@ -0,0 +1,37 @@ +METRICS = ["mean", "max", "min", "stddev"] +COLOR_SCHEME = 'coolwarm' + +GRAPH_STYLE = { + "rot":45, + "fontsize":6, + "legend":None, + "title": "Tokens per second (average)", + "x": "title", + "xlabel": "Model" +} + +SUPPORTED_FORMATS = ["free", "mpstat", "csv", "nvidia-smi"] + +OUTPUT_FORMAT = "png" + +MPSTAT_HEADER = "time\tCPU\t%usr\t%nice\t%sys\t%iowait\t%irq\t%soft\t%steal\t%guest\t%gnice\t%idle\n" + +MPSTAT_OUTPUT = ["title", "InfCPU", "MaxCPU"] +FREE_OUTPUT = ["title", "MaxMem", "InfMem"] +NVIDIA_SMI_OUTPUT = ["title","InfVRAM", "MaxVRAM", "InfVRAMBW%", "InfMaxSinglVRAMBW%", "InfGPU%", "InfMaxSinglGPU%" ] + +OUTPUT_SCHEMAS = { + "csv": ["title", *METRICS], + "free": FREE_OUTPUT, + "mpstat": MPSTAT_OUTPUT, + "nvidia-smi": NVIDIA_SMI_OUTPUT +} + +# Indicates the metric to highlight and the order of display +# dict[str, (metric, ascending?)] +HIGHLIGHTED_METRIC = { + "csv": ("first%", False), + "free": (FREE_OUTPUT[1], True), + "mpstat": (MPSTAT_OUTPUT[1], True), + "nvidia-smi": (NVIDIA_SMI_OUTPUT[1], True) +} diff --git a/tools/analytics/src/ec2benchmarks.py b/tools/analytics/src/ec2benchmarks.py new file mode 100644 index 0000000..b79ed33 --- /dev/null +++ b/tools/analytics/src/ec2benchmarks.py @@ -0,0 +1,34 @@ +import pandas as pd + +# from config import METRICS + +def __get_avg_tok_p_s(path: str): + df = pd.read_csv(path).drop("note", axis=1) + df["tok_per_sec"] = df["tok_count"] / df["time"] + data = df["tok_per_sec"] + return (data.mean(), data.max(), data.min(), data.std()) + + +# def process(files): +# data = [ (name[:-4], *__get_avg_tok_p_s(p)) for (p,name) in files ] +# df = pd.DataFrame(data, columns=["benchmark", *METRICS]).set_index("benchmark") +# df.sort_values(by=[METRICS[0]], inplace=True, ascending=False) +# first = max(df[METRICS[0]]) +# df["first%"] = df["mean"] / first * 100 +# +# return df + + + +def process(file): + # name = os.path.basename(file) + # data = (name[:-4], *__get_avg_tok_p_s(file)) + # df = pd.DataFrame(data, columns=["benchmark", *METRICS]).set_index("benchmark") + # df.sort_values(by=[METRICS[0]], inplace=True, ascending=False) + # first = max(df[METRICS[0]]) + # df["first%"] = df["mean"] / first * 100 + + # return data + return __get_avg_tok_p_s(file) + + diff --git a/tools/analytics/src/huggingface.py b/tools/analytics/src/huggingface.py new file mode 100644 index 0000000..dcc250d --- /dev/null +++ b/tools/analytics/src/huggingface.py @@ -0,0 +1,86 @@ +import pandas as pd +import numpy as np + + +def free(file, inf_mem_row=-3): + """ + file: name of the file to read + inf_mem_row: row to be used as peak inference memory usage + By default it's the third-to-last value + """ + df = pd.read_csv(file, sep="\t") + chunk_size = len(df)//2 + chunked = np.array_split(df["used"], chunk_size) + sums = np.empty(chunk_size) + for (i, chunk) in enumerate(chunked): + sums[i] = chunk.iloc[0].sum() + + inf_mem = sums[inf_mem_row] + return sums.max(), inf_mem + + +def mpstat(file, start=-13, end=-3): + """ + file: name of the file to read + start:end : time interval to consider, in seconds + The default range starts at the last 13 seconds and ends in the last 3 + + Metrics: + MaxCPU: maximal percentage of a single CPU usage during inference + InfCPU: average percentage of all CPUs usage during inference + """ + df = pd.read_csv(file, sep="\t") + # Mean of %idle + agg = df[ df["CPU"] == "all" ][["time", "%idle"]].iloc[start:end] + agg["%use"] = 100 - agg["%idle"] + infcpu = agg["%use"].mean() + + # maxcpu = 100 - df[ df["CPU"] == str(max_cpu_target) ]["%idle"].iloc[start:end].min() + group = df[ df["CPU"] != "all" ].groupby("time").min()["%idle"][start:end] + maxcpu = 100 - min(group) + return (infcpu, maxcpu) + + +def nvidia_smi(file, start=-13, end=-3): + """ + Keeping the header with units around for reference: + gpu (Idx) pwr (W) gtemp (C) mtemp (C) sm (%) mem (%) + enc (%) dec (%) jpg (%) ofa (%) mclk (MHz) pclk (MHz) + pviol (%) tviol (bool) fb (MB) bar1 (MB) ccpm (MB) + sbecc (errs) dbecc (errs) pci (errs) rxpci (MB/s) txpci (MB/s) + + file: name of the file to read + start:end : time interval to consider, in seconds + The default range starts at the last 13 seconds and ends in the last 3 + + Metrics: + InfGPU%: average of the per second average of sm% in the start:end time interval. + InfMaxSinglGPU%: max value of sm% among all GPUS in the start:end time interval + """ + df = pd.read_csv(file, sep="\t") + n_gpus = df["gpu"].nunique() + n_chunks = int(len(df) / n_gpus) + sums = np.empty(n_chunks) + + # As there are no time fields, separate the input in chunks, with a single entry in each chunk for every GPU + # Then keep only the metrics of interest + chunked = np.array_split(df[["fb", "sm", "mem"]], n_chunks) + + for (i, chunk) in enumerate(chunked): + sums[i] = chunk["fb"].sum() + infvram = sums[end] + maxvram = sums.max() + + # Average of the average of each chunk + # `abs(start-end)` is the duration of the chosen timeframe + infgpu = max([ + x for x in + map(lambda chunk: chunk["sm"].max(), chunked[start:end])]) #/ abs(start - end) + + infvram_bw_percent = max([x for x in map(lambda chunk: chunk["mem"].max(), chunked[start:end])]) + + infmaxsinglgpu_percent = max([ch["sm"].max() for ch in chunked[start:end]]) + infmaxsinglvrambw = max([ch["mem"].max() for ch in chunked[start:end]]) + + return (maxvram,infvram, infgpu, infvram_bw_percent, infmaxsinglgpu_percent, infmaxsinglvrambw) + diff --git a/tools/analytics/src/main.py b/tools/analytics/src/main.py new file mode 100644 index 0000000..d753b7a --- /dev/null +++ b/tools/analytics/src/main.py @@ -0,0 +1,78 @@ +import sys, os +from typing import List + +import seaborn as sns + +from render import AnalysisResults +from config import COLOR_SCHEME +from ec2benchmarks import process as process_ec2_data +from huggingface import free, mpstat, nvidia_smi +from preprocessing import preprocess_files + +def main(args): + directory = args[1] + out = args[2] + files = get_data_files(directory) + results = AnalysisResults(out) + results.set_destdir(out) + + for file in files: + ext = file.split(".")[-1] + match ext: + case "csv": + data = process_ec2_data(file) + results.add_entry(file, data) + + case "free": + (max_mem, inf_mem) = free(file) + results.add_entry(file, [max_mem, inf_mem]) + + case "mpstat": + (infcpu, maxcpu) = mpstat(file) + results.add_entry(file, [infcpu, maxcpu]) + + case "nvidia-smi": + ( + maxvram, + infvram, + infgpu, + infvram_bw_percent, + infmaxsinglgpu_percent, + infmaxsinglvrambw + ) = nvidia_smi(file, start=-20, end=-10) + + results.add_entry(file, [ + maxvram, + infvram, + infvram_bw_percent, + infmaxsinglvrambw, + infgpu, + infmaxsinglgpu_percent + ]) + + case "vmstat": + pass + + results.render_all() + + +def get_data_files(dir: str) -> List[str]: + paths = [] + walk = os.walk(dir) + for root,_,files in walk: + for f in files: + paths.append(f"{root}/{f}") + return paths + + +if __name__ == "__main__": + help_text = "Usage: \n\t py main.py [--preprocess] [directory]" + sns.set_theme(style="darkgrid", palette=COLOR_SCHEME) + if len(sys.argv) <= 1: + print(help_text) + exit(1) + if sys.argv[1] == "--preprocess": + preprocess_files(get_data_files(sys.argv[2])) + else: + main(sys.argv) + diff --git a/tools/analytics/src/postprocess.py b/tools/analytics/src/postprocess.py new file mode 100644 index 0000000..2972509 --- /dev/null +++ b/tools/analytics/src/postprocess.py @@ -0,0 +1,32 @@ +import pandas as pd + +from config import HIGHLIGHTED_METRIC, METRICS, NVIDIA_SMI_OUTPUT + + +def postprocess_csv(df: pd.DataFrame) -> pd.DataFrame: + df.sort_values(by=[METRICS[0]], inplace=True, ascending=False) + first = max(df[METRICS[0]]) + df["first%"] = df["mean"] / first * 100 + + return df + +def postprocess_free(df: pd.DataFrame) -> pd.DataFrame: + cols = [col for col in df.columns if col != "title"] + df[cols] = df[cols].astype(int) + (metric, _) = HIGHLIGHTED_METRIC["free"] + df.sort_values(by=metric, inplace=True, ascending=True) + return df + +def postprocess_mpstat(df: pd.DataFrame) -> pd.DataFrame: + (metric, _) = HIGHLIGHTED_METRIC["mpstat"] + df.sort_values(by=metric, inplace=True, ascending=True) + return df + + +def postprocess_nvidia_smi(df: pd.DataFrame) -> pd.DataFrame: + df = df[NVIDIA_SMI_OUTPUT] + int_cols = ["MaxVRAM", "InfVRAM", "InfGPU%", "InfVRAMBW%"] + df[int_cols] = df[int_cols].astype(int) + (metric, _) = HIGHLIGHTED_METRIC["nvidia-smi"] + df.sort_values(by=metric, inplace=True, ascending=True) + return df diff --git a/tools/analytics/src/preprocessing.py b/tools/analytics/src/preprocessing.py new file mode 100644 index 0000000..262dab8 --- /dev/null +++ b/tools/analytics/src/preprocessing.py @@ -0,0 +1,80 @@ +import os, sys +from typing import List + +import shutil + +from config import MPSTAT_HEADER + + +def preprocess_files(files: List[str]): + for file in files: + parent_dir = os.path.dirname(file).split("/")[0] + rest = os.path.dirname(file).split("/")[1:] + outdir = parent_dir + "_preprocessed/" + "/".join(rest) + if not os.path.exists(outdir): + os.makedirs(outdir, exist_ok=True) + filename = os.path.basename(file) + dst = f"{outdir}/{filename}" + os.makedirs(os.path.dirname(dst), exist_ok=True) + + match file.split(".")[-1]: + case "mpstat": + result = _preprocess_mpstat(file) + case "free": + result = _preprocess_free(file) + case "nvidia-smi": + result = _preprocess_nvidia_smi(file) + case "csv": + # Already correct formatting, just copy it over + shutil.copy(file, dst) + continue + case _: + print(f"[WARNING] Unsupported file format: {dst}", file=sys.stderr) + continue + + with open(dst, "w+") as f: + f.write(result) + + +def _preprocess_mpstat(file) -> str: + with open(file) as f: + data = f.readlines() + # Filter out blank lines and useless or redundant headers + keep = [ + _replace_spaces_with_tabs(line) + for line in data + if len(line) > 0 and "CPU" not in line and not line.startswith("Linux") + ] + keep.insert(0, MPSTAT_HEADER) + return "".join(keep) + + +def _preprocess_free(file) -> str: + with open(file) as f: + lines = f.readlines() + header = lines[0] + rest = lines[1:] + data = "".join([ + _replace_spaces_with_tabs(l) + for l in rest + if "free" not in l + ]) + # the column which indicates Mem or Swap lacks a name, call it type and add it to the header + return f"type\t{_replace_spaces_with_tabs(header)}{data}" + +def _preprocess_nvidia_smi(file) -> str: + with open(file) as f: + lines = f.readlines() + # Remove the two first lines that start with '#' + header = lines[0][2:] + # Skip units + rest = lines[2:] + data = [ + _replace_spaces_with_tabs(l) + for l in rest + if "#" not in l + ] + return _replace_spaces_with_tabs(header) + "".join(data) + +def _replace_spaces_with_tabs(text): + return "\t".join([x for x in text.split(" ") if x != ""]) diff --git a/tools/analytics/src/render.py b/tools/analytics/src/render.py new file mode 100644 index 0000000..5f62f49 --- /dev/null +++ b/tools/analytics/src/render.py @@ -0,0 +1,98 @@ +import os +from typing import Dict +from jinja2 import Environment, FileSystemLoader +import dataframe_image as dfi +import matplotlib.pyplot as plt +import pandas as pd + +from config import ( + COLOR_SCHEME, + GRAPH_STYLE, + HIGHLIGHTED_METRIC, + METRICS, + OUTPUT_FORMAT, + OUTPUT_SCHEMAS, +) +from postprocess import postprocess_csv, postprocess_free, postprocess_mpstat, postprocess_nvidia_smi + + +class AnalysisResults: + results: Dict[str, pd.DataFrame] + # High to low + color_h2l = COLOR_SCHEME + # Low to high + color_l2h = COLOR_SCHEME + "_r" + + def __init__(self, destdir: str) -> None: + self.results = dict() + self.destdir = destdir + for format, schema in OUTPUT_SCHEMAS.items(): + self.results[format] = pd.DataFrame(columns=schema) + + def set_destdir(self, dst): + self.destdir = dst + + def __repr__(self) -> str: + result = [] + for format, data in self.results.items(): + result.append(f"{format}:\n {data.head(10)}") + return "\n".join(result) + + def __render_page(self): + graph_file = f"{self.destdir}/graph.{OUTPUT_FORMAT}" + generated_files = [graph_file] + + for datafmt, data in self.results.items(): + if data.empty: + continue + (hl_metric, h2l) = HIGHLIGHTED_METRIC[datafmt] + # float_cols = [col for col in data.columns if col != "title" and col != hl_metric ] + # data[float_cols] = data[float_cols].map('{:.2f}'.format)#.astype(float) + cmap = self.color_h2l if h2l else self.color_l2h + styled = data.style \ + .background_gradient(subset=[hl_metric], cmap=cmap) \ + .hide() \ + .format(precision=1, thousands="", decimal=".") + + filename = f"{self.destdir}/table_{datafmt}.{OUTPUT_FORMAT}" + print(f"Saving in {self.destdir}/table_{datafmt}.{OUTPUT_FORMAT}") + dfi.export( + # data.style.background_gradient(subset=["first%"], cmap=COLOR_SCHEME), + # .format("{:,.2f}".format), + styled, + filename, + table_conversion="matplotlib", + ) + generated_files.append(filename) + + environment = Environment(loader=FileSystemLoader("templates/")) + template = environment.get_template("graph.html") + content = template.render( + files=[os.path.abspath(f) for f in generated_files] + ) + with open(f"{self.destdir}/index.html", "w+") as f: + f.write(content) + plt.savefig(graph_file, bbox_inches="tight", dpi=400) + + def render_all(self): + self.__postprocess_data() + metric = METRICS[0] + self.results["csv"].plot.bar(y=metric, **GRAPH_STYLE) + if not os.path.exists(self.destdir): + os.mkdir(self.destdir) + self.__render_page() + + def add_entry(self, filename: str, row): + """ + Used to add one row at a time + """ + file = filename.split("/")[-1] + title = file.split(".")[0] + ext = file.split(".")[-1] + self.results[ext].loc[len(self.results[ext])] = [title, *row] + + def __postprocess_data(self): + self.results["csv"] = postprocess_csv(self.results["csv"]) + self.results["free"] = postprocess_free(self.results["free"]) + self.results["mpstat"] = postprocess_mpstat(self.results["mpstat"]) + self.results["nvidia-smi"] = postprocess_nvidia_smi(self.results["nvidia-smi"]) diff --git a/tools/analytics/templates/graph.html b/tools/analytics/templates/graph.html new file mode 100644 index 0000000..7a25664 --- /dev/null +++ b/tools/analytics/templates/graph.html @@ -0,0 +1,35 @@ + + + + + Hello, world! + + + + + + + {% for file in files %} + + {% endfor %} + + + + diff --git a/tools/analytics/templates/table.html b/tools/analytics/templates/table.html new file mode 100644 index 0000000..2cad6fd --- /dev/null +++ b/tools/analytics/templates/table.html @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
meanmaxminstddevfirst%
34.1634.3233.340.33100.00
33.1233.1633.060.0396.95
32.7641.1528.183.9795.88
32.7241.1727.944.0295.79
32.4632.5032.370.0595.02
31.6432.2227.991.4892.63
26.7032.116.678.5278.15
25.4430.0716.314.9974.47
25.2428.4017.973.6873.88
22.1826.652.008.7064.94
20.5026.6110.285.4660.01
20.3526.6112.425.8259.58
\ No newline at end of file