initial commit to push copy to codeberg

changelog2/__main__.py

@@ -0,0 +1,4 @@
+from .main import main
+
+if __name__ == "__main__":
+    main()

changelog2/config/agents.yaml

@@ -0,0 +1,29 @@
+gitlog_analyst:
+  role: >
+    Git Log Analyst
+  goal: >
+    Collect all relevant informations from Git logs
+  backstory: >
+   You specialize in analyzing Git Logs. You extract important Information such as commit messages,
+    dates and categorize the commits by new, changed, and fixed based on their commit message.
+
+redmine_analyst:
+  role: >
+    Redmine Analyst
+  goal: >
+    Gather information from Redmine tickets
+  backstory: >
+    You are an expert in Redmine. You collect relevant information from Redmine tickets for the changelog
+
+writing_agent:
+  role: >
+    Technical Writer
+  goal: >
+    Transform technical information into clear, concise, and user-friendly documentation
+  backstory: >
+    You are an experienced technical writer with a strong background in software development
+    and documentation. Your expertise lies in translating complex technical concepts into
+    easily understandable content for various audiences, from end-users to developers.
+    You have a keen eye for detail and a passion for creating documentation that enhances
+    user experience and product understanding.
+

changelog2/config/config.py

@@ -0,0 +1,34 @@
+import configparser
+import os
+from pathlib import Path
+
+
+def find_config():
+    possible_locations = [
+        Path.cwd() / "changelog-config.toml",
+        Path.cwd() / "changelog2" / "changelog-config.toml",
+        Path.cwd() / ".config" / "changelog-config.toml",
+        Path.cwd() / ".config" / "changelog-config.toml",
+        Path.home() / "changelog-config.toml",
+        Path(os.getenv("CHANGELOG_CONFIG", "")),
+    ]
+
+    for location in possible_locations:
+        if location.is_file():
+            return str(location)
+
+    raise FileNotFoundError("changelog-config.toml nicht gefunden")
+
+
+config_path = find_config()
+
+
+def parse_config():
+    config = configparser.ConfigParser()
+    config.read(config_path)
+    return {
+        "REDMINE_HOST": config["DEFAULT"]["REDMINE_HOST"],
+        "REDMINE_API_KEY": config["DEFAULT"]["REDMINE_API_KEY"],
+        "OPENAI_API_KEY": config["DEFAULT"]["OPENAI_API_KEY"],
+        "GIT_REPO_PATH": config["DEFAULT"]["GIT_REPO_PATH"],
+    }

changelog2/config/tasks.yaml

@@ -0,0 +1,47 @@
+gitlog_analysis_task:
+  description: >
+    Collect all relevant Git commits since the last tag for the projects {project_id} which are seperated by ",".
+    Analyze the commits and categorize them according to new features, changes, and fixes.
+
+    Git Repo: {repo_path}
+  expected_output: >
+    A JSON string containing a list of relevant Git commits with their hash, date, message, a {changelog_type} entry understandable for end users
+    and category (new feature, change, or fix).
+
+redmine_analysis_task:
+  description: >
+    Collect all relevant Redmine tickets for the projects {project_id} and their versions {version_id}.
+    Analyze the tickets and categorize them based on their type and importance.
+  expected_output: >
+    A JSON string containing a list of relevant Redmine tickets with their ID, subject, tracker,
+    a {changelog_type} entry understandable for end users and a brief summary of their content and importance.
+
+formatting_task:
+  description: >
+    Format the collected Git commits and Redmine tickets into a structured {changelog_type}.
+    Ensure that the information is organized logically and easy to read.
+  expected_output: >
+    A well-formatted {changelog_type} text that includes both Git commits and Redmine tickets
+    in a clear and organized manner, with proper categorization and highlighting of key changes.
+
+writing_task:
+  description: >
+    Using the information from gitlog and redmine tickets, create a user-friendly and informative {changelog_type} document.
+    Translate technical details into clear, concise language suitable for both technical
+    and non-technical readers. Highlight key features, changes, and fixes.
+  expected_output: >
+    A polished {changelog_type} document that effectively communicates all relevant changes,
+    improvements, and fixes in a way that enhances user understanding and engagement.
+    The document should maintain consistency with the rules provided by the RAG tool and should fit the template from the template_tool
+    while accurately reflecting the current changelog's content.
+  instructions: >
+    1. Review the template provided by the template tool.
+    2. Identify the key sections and formatting used.
+    3. Analyze the content of the gitlog and redmine issues.
+    4. Structure the new changelog following the templates format, including similar sections.
+    5. Translate technical details into user-friendly language.
+    6. Ensure all significant changes, features, and fixes are included and highlighted.
+    7. Maintain a consistent tone and style throughout the document.
+    8. Double-check that the final document is informative, clear, and engaging for all readers.
+    9. Skip issues/log entries not relevant for the user
+    10. Do not include any additional output

changelog2/crew.py

@@ -0,0 +1,153 @@
+import os
+from typing import List
+
+from config.config import parse_config
+from crewai import Agent, Crew, Process, Task
+from crewai.project import CrewBase, agent, crew, task
+from crewai.tasks.conditional_task import ConditionalTask
+from crewai_tools import FileReadTool
+from langchain_ollama import ChatOllama
+from langchain_openai import ChatOpenAI
+from pydantic import BaseModel
+from tools.tools import (
+    get_categorized_commits_since_last_tag,
+    get_commits_since_last_tag,
+    get_redmine_issues,
+)
+
+config = parse_config()
+os.environ["OPENAI_API_KEY"] = config["OPENAI_API_KEY"]
+rag_tool = FileReadTool(file_path="./data/CodeOfConduct.md")
+template_tool = FileReadTool(file_path="./data/template2.txt")
+llmLocal = ChatOllama(model="gemma2", base_url="http://ollama:11434")
+llmOpenAI = ChatOpenAI(
+    base_url="https://api.openai.com/v1/",
+    model_name="gpt-4o",
+    temperature=0.4,
+)
+
+
+@CrewBase
+class ChangelogCrew:
+    """Changelog writing crew"""
+
+    agents_config = "config/agents.yaml"
+    tasks_config = "config/tasks.yaml"
+
+    def __init__(self, git_task: bool = False, local: bool = True):
+        self.git_task = git_task
+        self.local = local
+        if self.local:
+            os.environ["OPENAI_API_BASE"] = "http://ollama:11434"
+            os.environ["OPENAI_MODEL_NAME"] = "gemma2"
+        else:
+            os.environ["OPENAI_API_BASE"] = "https://api.openai.com/v1/chat/completions"
+
+    @agent
+    def gitlog_analyst(self) -> Agent:
+        if self.local:
+            llm = llmLocal
+        else:
+            llm = llmOpenAI
+        return Agent(
+            config=self.agents_config["gitlog_analyst"],
+            tools=[get_categorized_commits_since_last_tag],
+            llm=llm,
+            max_iter=5,
+            verbose=False,
+            memory=False,
+        )
+
+    @agent
+    def redmine_analyst(self) -> Agent:
+        if self.local:
+            llm = llmLocal
+        else:
+            llm = llmOpenAI
+        return Agent(
+            config=self.agents_config["redmine_analyst"],
+            tools=[get_redmine_issues],
+            llm=llm,
+            max_iter=5,
+            verbose=False,
+            memory=False,
+        )
+
+    @agent
+    def writing_agent(self) -> Agent:
+        if self.local:
+            llm = llmLocal
+        else:
+            llm = llmOpenAI
+        with open("./data/template2.txt", "r") as template_file:
+            template_content = template_file.read()
+        return Agent(
+            config=self.agents_config["writing_agent"],
+            tools=[template_tool],
+            llm=llm,
+            max_iter=5,
+            verbose=False,
+            memory=False,
+            template=template_content,
+        )
+
+    @task
+    def redmine_analysis_task(self) -> Task:
+        return Task(
+            config=self.tasks_config["redmine_analysis_task"],
+            agent=self.redmine_analyst(),
+        )
+
+    @task
+    def gitlog_analysis_task(self) -> ConditionalTask:
+        return ConditionalTask(
+            config=self.tasks_config["gitlog_analysis_task"],
+            condition=lambda context: self.git_task,
+            agent=self.gitlog_analyst(),
+        )
+
+    @task
+    def writing_task(self) -> Task:
+        context = []
+        context.append(self.redmine_analysis_task())
+        if self.git_task:
+            context.append(self.gitlog_analysis_task())
+
+        return Task(
+            config=self.tasks_config["writing_task"],
+            agent=self.writing_agent(),
+            context=context,
+        )
+
+    @crew
+    def crew(self) -> Crew:
+        """Creates the Changelog crew"""
+        return Crew(
+            agents=self.agents,
+            tasks=self.tasks,
+            process=Process.sequential,
+            verbose=False,
+        )
+
+
+class GitCommit(BaseModel):
+    hash: str
+    author: str
+    date: str
+    message: str
+    category: str
+
+
+class RedmineTicket(BaseModel):
+    id: int
+    subject: str
+    tracker: str
+    summary: str
+
+
+class Changelog(BaseModel):
+    version: str
+    date: str
+    changes: List[str]
+    fixes: List[str]
+    new_features: List[str]

changelog2/data/CodeOfConduct.md

@@ -0,0 +1,57 @@
+# Code of Conduct für Änderungsprotokolle
+
+## Allgemeine Richtlinien
+
+1. Klarheit und Konsistenz: Alle Einträge müssen klar und konsistent formuliert sein, um Verwirrung zu vermeiden. Verwenden Sie klare, präzise Sprache und folgen Sie den unten aufgeführten Formatierungsrichtlinien.
+2. Abkürzungen und Akronyme: Verwenden Sie standardisierte Abkürzungen für Module und Komponenten. Eine Liste der häufig verwendeten Abkürzungen finden Sie im Abschnitt "Abkürzungen".
+3. Versionsnummern: Geben Sie die Versionsnummern der betroffenen Komponenten klar an. Verwenden Sie die Formatierung Komponente: alte Version -> neue Version.
+4. Kategorien von Änderungen: Gliedern Sie Änderungen in die Kategorien NEW, CHANGED, FIXED, und DOCUMENTATION. Jede Kategorie sollte klar vom Rest des Dokuments abgegrenzt sein.
+
+## Struktur und Formatierung
+
+### Titel und Metadaten
+
+Titel: Verwenden Sie das Format = [Produktname] - Changelog. Beispiel: = TIXstream FX - Changelog.
+Website und Sprache: Geben Sie die Website und die Sprache an. Beispiel:
+
+
+### Copyright und Versionsinfo
+
+- Copyright: Fügen Sie das Copyright-Dokument ein. Beispiel:
+
+
+- Versionsinfo: Geben Sie die Versionsinformationen ein. Beispiel:
+
+
+### Änderungsprotokoll
+
+- Einführung: Erklären Sie den Zweck des Dokuments und die Bedeutung der einzelnen Einträge. Beispiel:
+
+
+- Versionseinträge: Jede Version sollte mit der Versionsnummer und dem Datum beginnen. Beispiel:
+
+
+- Neue Funktionen (NEW): Listen Sie neue Funktionen und Features auf. Verwenden Sie nummerierte Punkte oder Aufzählungen.
+- Änderungen (CHANGED): Dokumentieren Sie wesentliche Änderungen an bestehenden Funktionen oder Systemen.
+- Fehlerbehebungen (FIXED): Beschreiben Sie behobene Fehler und Probleme.
+- Dokumentation (DOCUMENTATION): Erwähnen Sie Änderungen an der Dokumentation oder neuen Dokumenten.
+
+### Eintragsformatierung
+
+- Klarheit der Beschreibungen: Beschreibungen sollten so präzise wie möglich sein. Vermeiden Sie unnötigen Jargon und halten Sie die Erklärungen einfach.
+- Verweise auf Tickets: Falls zutreffend, verwenden Sie Ticket-IDs zur Referenz. Beispiel: (_#12345_).
+- Modul- und Komponentennamen: Verwenden Sie die offiziellen Namen und Abkürzungen für Komponenten. Beispiel:
+
+
+### Abkürzungen
+
+Use standard abbreviations for components:
+
+- AM: Access Manager
+- FXweb: FX Web Tools
+- TCC: TIXEL Control Center
+- TXEC: TIXstream Express Client
+- TXEJM: TIXstream Express Job Manager
+- TXSFX: TIXstream FX
+- TXY: TIXway
+

changelog2/data/template2.txt

@@ -0,0 +1,57 @@
+= {Produktname} - Changelog
+:website: http://www.example.com
+:lang: en
+:encoding: utf-8
+
+.COPYRIGHT
+***********************************************************************
+include::../shared-doc/src/copyright.adoc[]
+***********************************************************************
+
+Software Version -- Document Version:
+include::version.string[]
+
+== Redmine Issues
+
+// Hier können Sie die Redmine-Issues auflisten
+// Beispiel:
+* #12345 - Beschreibung des Issues
+* #12346 - Beschreibung des Issues
+* #12347 - Beschreibung des Issues
+
+== Changelog
+
+This document lists all relevant changes applied over time to {Produktname} packages and their software components.
+Each entry in the list shows changes made to the *Services*, the installation *Bundle* or the *Documentation*.
+Entries may have an _ID_ at the end of the line for internal reference.
+
+== Version {Versionsnummer} ({Datum})
+
+*NEW*
+
+{{#each ticket or log entry}}
+=== _({{id/hash}})_ {{subject}} ===
+* {{Your generated changelog entry}} _({{ticket_id/git hash}})_
+{{/each}}
+
+*CHANGED*
+
+{{#each ticket or log entry}}
+=== _({{id/hash}})_ {{subject}} ===
+* {{Your generated changelog entry}} _({{ticket_id/git hash}})_
+{{/each}}
+
+*FIXED*
+
+{{#each ticket or log entry}}
+=== _({{id/hash}})_ {{subject}} ===
+* {{Your generated changelog entry}} _({{ticket_id/git hash}})_
+{{/each}}
+
+*DOCUMENTATION*
+
+{{#each ticket or log entry}}
+=== _({{id/hash}})_ {{subject}} ===
+* {{Your generated changelog entry}} _({{ticket_id/git hash}})_
+{{/each}}
+

changelog2/gittest.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if ! git describe --tags --abbrev=0 &>/dev/null; then
+    echo "Keine Tags gefunden. Verwende den ersten Commit als Startpunkt."
+    START_POINT=$(git rev-list --max-parents=0 HEAD)
+else
+    START_POINT=$(git describe --tags --abbrev=0)
+fi
+
+echo "Changelog seit $START_POINT:"
+echo ""
+echo "Neue Features:"
+git log --no-merges --pretty=format:"- %s" $START_POINT..HEAD | grep "^feat:" | sort
+echo ""
+echo "Fehlerbehebungen:"
+git log --no-merges --pretty=format:"- %s" $START_POINT..HEAD | grep "^fix:" | sort
+echo ""
+echo "Dokumentation:"
+git log --no-merges --pretty=format:"- %s" $START_POINT..HEAD | grep "^docs:" | sort
+echo ""
+echo "Andere Änderungen:"
+git log --no-merges --pretty=format:"- [%h] %s (%an)" $START_POINT..HEAD | grep -vE "^- (feat|fix|docs):" | sort

changelog2/main.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+import argparse
+import json
+import os
+import ssl
+import urllib.request as request
+
+from config.config import parse_config
+from crew import ChangelogCrew
+from langchain_ollama import ChatOllama
+from langchain_openai import ChatOpenAI
+
+config = parse_config()
+os.environ["OPENAI_API_KEY"] = config["OPENAI_API_KEY"]
+
+# os.environ["OPENAI_API_BASE"] = "http://ollama:11434"
+# os.environ["OPENAI_MODEL_NAME"] = "llama3.1:8b"  # Adjust based on available model
+
+llmLocal = ChatOllama(
+    model="llama3.1:8b", base_url="http://ollama:11434", temperatur=0.4
+)
+llmOpenAI = ChatOpenAI(
+    model_name="gpt-4o",
+    temperature=0.4,
+)
+config["git_task"] = False
+
+
+def makeRedmineRequest(url: str, APIKey: str):
+    context = ssl.create_default_context()
+    context.check_hostname = False
+    context.verify_mode = ssl.CERT_NONE
+
+    req = request.Request(url, method="GET")
+    req.add_header("Content-Type", "application/json")
+    req.add_header("X-Redmine-API-Key", APIKey)
+    return request.urlopen(req, context=context)
+
+
+def makeProjectRequest(url: str, APIKey: str) -> dict:
+    res = makeRedmineRequest(url, APIKey)
+    json_data = json.load(res)
+    project_info = {project["name"]: project["id"] for project in json_data["projects"]}
+
+    return project_info
+
+
+def makeVersionRequest(url: str, APIKey: str, project_id: int) -> dict:
+    res = makeRedmineRequest(url, APIKey)
+    json_data = json.load(res)
+
+    version_info = {
+        version["name"]: version["id"]
+        for version in json_data["versions"]
+        if version["project"]["id"] == project_id and version["status"] == "open"
+    }
+
+    return version_info
+
+
+def main():
+    desc = "This is a tool that generates changelog messages from Redmine and Git using AI."
+    parser = argparse.ArgumentParser(description=desc, add_help=True)
+    parser.add_argument(
+        "-p",
+        "--project",
+        help="The project ID for which text is to be generated",
+        # type=int,
+        type=str,
+    )
+    parser.add_argument(
+        "-f",
+        "--fixed-version",
+        help="The version ID of the project for which text is to be generated",
+        # type=int,
+        type=str,
+    )
+    parser.add_argument(
+        "-r",
+        "--repo",
+        help="The path to the Git repository",
+        type=str,
+    )
+    parser.add_argument(
+        "-t",
+        "--type",
+        help="The type of text to be generated",
+        choices=["Changelog", "ReleaseNotes", "test"],
+        type=str,
+    )
+    parser.add_argument(
+        "-l",
+        "--local",
+        help="Execute the generator using a local llm",
+        default=False,
+        action="store_true",
+    )
+    args = parser.parse_args()
+    if not args.project:
+        project_infos = makeProjectRequest(
+            url=config["REDMINE_HOST"] + "/projects.json",
+            APIKey=config["REDMINE_API_KEY"],
+        )
+        print("Enter a project id when calling. There is a choice:")
+        for project_name, project_id in project_infos.items():
+            print(f"{project_name} (ID: {project_id})")
+        exit(0)
+    if args.project and not args.fixed_version:
+        print("Enter a version id when calling. There is a choice:")
+        fixed_version = makeVersionRequest(
+            url=config["REDMINE_HOST"]
+            + "/projects/"
+            + str(args.project)
+            + "/versions.json",
+            APIKey=config["REDMINE_API_KEY"],
+            project_id=args.project,
+        )
+        for version_name, version_id in fixed_version.items():
+            print(f"{version_name}: {version_id}")
+        exit(0)
+    if args.project and args.fixed_version and not args.type:
+        args.type = input("Please choose a type (Changelog/ReleaseNotes)")
+        customField = ""
+        prompt = ""
+    if args.project and args.fixed_version:
+        if args.type == "Changelog":
+            customField = "&cf_21=Changelog"
+        elif args.type == "ReleaseNotes":
+            customField = "&cf_21=Release_notes"
+        else:
+            customField = ""
+
+    if args.repo != None:
+        os.environ["GIT_REPO_PATH"] = args.repo
+        config["git_task"] = True
+    else:
+        os.environ["GIT_REPO_PATH"] = ""
+
+    changelog_crew = ChangelogCrew(git_task=config["git_task"], local=args.local)
+    result = changelog_crew.crew().kickoff(
+        inputs={
+            "project_id": args.project,
+            "version_id": args.fixed_version,
+            "repo_path": args.repo,
+            "changelog_type": args.type,
+        }
+    )
+
+    print("######################")
+    print(result)
+
+
+if __name__ == "__main__":
+    main()

changelog2/tools/tools.py

@@ -0,0 +1,159 @@
+import json
+import re
+import ssl
+import urllib.request as request
+from typing import Dict, List
+
+import git
+from config.config import parse_config
+from crewai_tools import tool
+
+config = parse_config()
+
+
+@tool("Get Categorized Git Commits Since Last Tag")
+def get_categorized_commits_since_last_tag(
+    repo_path: str = config["GIT_REPO_PATH"],
+) -> str:
+    """
+    Retrieves and categorizes commits since the last tag in the Git repository using git shortlog.
+
+    Args:
+    repo_path (str): Path to the Git repository.
+
+    Returns:
+    str: JSON string containing categorized commit information.
+    """
+    repo = git.Repo(repo_path)
+    tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
+    last_tag = tags[-1] if tags else None
+
+    if not last_tag:
+        range_spec = "HEAD"
+    else:
+        range_spec = f"{last_tag.name}..HEAD"
+
+    shortlog = repo.git.shortlog("-sne", range_spec)
+
+    author_commits = parse_shortlog(shortlog)
+
+    commits = list(repo.iter_commits(range_spec))
+
+    categorized_commits = categorize_commits(commits)
+
+    result = {
+        "last_tag": last_tag.name if last_tag else "No tags",
+        "author_commits": author_commits,
+        "categorized_commits": categorized_commits,
+    }
+
+    return json.dumps(result, indent=2)
+
+
+def parse_shortlog(shortlog: str) -> List[Dict[str, str]]:
+    """Parse git shortlog output."""
+    author_commits = []
+    for line in shortlog.split("\n"):
+        if line.strip():
+            count, author = line.strip().split("\t")
+            author_commits.append(
+                {"commit_count": int(count.strip()), "author": author.strip()}
+            )
+    return author_commits
+
+
+def categorize_commits(commits: List[git.Commit]) -> Dict[str, List[Dict[str, str]]]:
+    """Categorize commits based on their message."""
+    categories = {"features": [], "fixes": [], "documentation": [], "others": []}
+
+    for commit in commits:
+        commit_info = {
+            "hash": commit.hexsha[:7],
+            "message": commit.summary,
+            "date": commit.committed_datetime.isoformat(),
+        }
+
+        if commit.summary.startswith("feat:"):
+            categories["features"].append(commit_info)
+        elif commit.summary.startswith("fix:"):
+            categories["fixes"].append(commit_info)
+        elif commit.summary.startswith("docs:"):
+            categories["documentation"].append(commit_info)
+        else:
+            categories["others"].append(commit_info)
+
+    return categories
+
+
+@tool("Get Git Commits Since Last Tag")
+def get_commits_since_last_tag(repo_path: str = config["GIT_REPO_PATH"]) -> str:
+    """
+    Retrieves all commits since the last tag in the Git repository.
+
+    Args:
+    repo_path (str): Path to the Git repository.
+
+    Returns:
+    str: JSON string containing commit information.
+    """
+    repo = git.Repo(repo_path)
+    tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
+    last_tag = tags[-1] if tags else None
+
+    if not last_tag:
+        commits = list(repo.iter_commits())
+    else:
+        commits = list(repo.iter_commits(f"{last_tag.name}..HEAD"))
+
+    commit_info = [
+        {
+            "hash": commit.hexsha,
+            "date": commit.committed_datetime.isoformat(),
+            "message": commit.message.strip(),
+        }
+        for commit in commits
+    ]
+    return json.dumps(commit_info)
+
+
+@tool("Get Redmine Issues")
+def get_redmine_issues(project_id: int, version_id: int, changelog_type: str) -> str:
+    """
+    Retrieves issues for a specific Redmine project and version.
+
+    Args:
+    project_id (int): ID of the Redmine project.
+    version_id (int): ID of the project version.
+    changelog_type (str): Type of changelog ("Changelog" or "Release_notes").
+
+    Returns:
+    str: JSON string containing issue information.
+    """
+    if changelog_type == "Changelog":
+        custom_field = "&cf_21=Changelog"
+    elif changelog_type == "ReleaseNotes":
+        custom_field = "&cf_21=Release_notes"
+    else:
+        custom_field = ""
+    url = f"{config['REDMINE_HOST']}/issues.json?project_id={project_id}&fixed_version_id={version_id}&status_id=*{custom_field}"
+    data = make_redmine_request(url)
+    issues = [
+        {
+            "id": issue["id"],
+            "subject": issue["subject"],
+            "tracker": issue["tracker"]["name"],
+        }
+        for issue in data["issues"]
+    ]
+    return json.dumps(issues)
+
+
+def make_redmine_request(url):
+    context = ssl.create_default_context()
+    context.check_hostname = False
+    context.verify_mode = ssl.CERT_NONE
+
+    req = request.Request(url, method="GET")
+    req.add_header("Content-Type", "application/json")
+    req.add_header("X-Redmine-API-Key", config["REDMINE_API_KEY"])
+    return json.load(request.urlopen(req, context=context))