Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Flake8 code style linter #19

Merged
merged 1 commit into from
Apr 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,14 @@ The [tox][https://github.com/tox-dev/tox] is used to manage tests. Please instal

This will create a virtual environment with dependencies and run all the tests. For more information follow the tox help.

To run only a specific test execute this:

tox run -e style # to run flake8

or

tox run -e lint # to run pylint

License
-------

Expand Down
16 changes: 7 additions & 9 deletions logdetective/logdetective.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def __init__(self):

def __call__(self, block_num, block_size, total_size):
if not self.pbar:
self.pbar=progressbar.ProgressBar(maxval=total_size)
self.pbar = progressbar.ProgressBar(maxval=total_size)
self.pbar.start()

downloaded = block_num * block_size
Expand All @@ -74,7 +74,7 @@ def get_chunks(text: str):
while i < text_len:
chunk += text[i]
if text[i] == '\n':
if i+1 < text_len and (text[i+1].isspace() or text[i-1] == "\\"):
if i + 1 < text_len and (text[i + 1].isspace() or text[i - 1] == "\\"):
i += 1
continue
yield chunk
Expand All @@ -101,23 +101,22 @@ def __call__(self, log: str, n_lines: int = 2, neighbors: bool = False) -> str:

def rate_chunks(self, log: str, n_lines: int = 2) -> list[tuple]:
"""Scan log by the model and store results.

:param log: log file content
:param n_lines: How many lines should the model take into consideration
"""
results = []
log_lines = log.split("\n")

for i in range(0, len(log_lines), n_lines):
block = '\n'.join(log_lines[i:i+n_lines])
block = '\n'.join(log_lines[i:i + n_lines])
prompt = SUMMARIZE_PROPT_TEMPLATE.format(log)
out = self.model(prompt, max_tokens=7, grammar=self.grammar)
out = f"{out['choices'][0]['text']}\n"
results.append((block, out))

return results


def create_extract(self, chunks: list[tuple], neighbors: bool = False) -> str:
"""Extract interesting chunks from the model processing.
"""
Expand All @@ -128,7 +127,7 @@ def create_extract(self, chunks: list[tuple], neighbors: bool = False) -> str:
if chunks[i][1].startswith("Yes"):
interesting.append(i)
if neighbors:
interesting.extend([max(i-1, 0), min(i+1, len(chunks)-1)])
interesting.extend([max(i - 1, 0), min(i + 1, len(chunks) - 1)])

interesting = np.unique(interesting)

Expand Down Expand Up @@ -205,13 +204,12 @@ def main():
parser.add_argument("url", type=str, default="")
parser.add_argument("-M", "--model", type=str, default=DEFAULT_ADVISOR)
parser.add_argument("-S", "--summarizer", type=str, default="drain")
parser.add_argument("-N","--n_lines", type=int, default=5)
parser.add_argument("-N", "--n_lines", type=int, default=5)
parser.add_argument("-v", "--verbose", action='count', default=0)
parser.add_argument("-q", "--quiet", action='store_true')

args = parser.parse_args()


if args.verbose and args.quiet:
sys.stderr.write("Error: --quiet and --verbose is mutually exclusive.\n")
sys.exit(2)
Expand Down Expand Up @@ -248,7 +246,7 @@ def main():
log = requests.get(args.url, timeout=60).text
log_summary = extractor(log)

ratio = len(log_summary.split('\n'))/len(log.split('\n'))
ratio = len(log_summary.split('\n')) / len(log.split('\n'))
LOG.debug("Log summary: \n %s", log_summary)
LOG.info("Compression ratio: %s", ratio)

Expand Down
12 changes: 12 additions & 0 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,21 @@
min_version = 4.0
env_list =
lint
style

[testenv:lint]
description = run Pylint
deps =
pylint>=3.0.0
commands = pylint {posargs:./logdetective/logdetective.py}

[testenv:style]
description = run Flake8
deps =
flake8>=7.0.0
commands = flake8 {posargs:./logdetective/logdetective.py}

[flake8]
ignore =
# disable max line leght, that is already checked by pylint
E501
Loading