reduce max tokens; refresh requirements.txt

requirements.txt

@@ -1,203 +0,0 @@
-accelerate==1.6.0
-aiohappyeyeballs==2.6.1
-aiohttp==3.11.18
-aiosignal==1.3.2
-annotated-types==0.7.0
-anyio==4.9.0
-attrs==25.3.0
-avidtools==0.1.2
-backoff==2.2.1
-base2048==0.1.3
-blinker==1.9.0
-boto3==1.38.2
-botocore==1.38.2
-cachetools==5.5.2
-certifi==2025.1.31
-cffi==1.17.1
-charset-normalizer==3.4.1
-chevron==0.14.0
-click==8.1.8
-cmd2==2.4.3
-cohere==4.57
-colorama==0.4.6
-coloredlogs==15.0.1
-dataclasses-json==0.6.7
-datasets==2.16.1
-DateTime==5.5
-deepl==1.17.0
-dill==0.3.7
-distro==1.9.0
-ecoji==0.1.1
-faiss-cpu==1.11.0
-fastapi==0.115.12
-fastavro==1.10.0
-filelock==3.18.0
-Flask==3.1.1
-flatbuffers==25.2.10
-frozenlist==1.6.0
-fschat==0.2.36
-fsspec==2023.10.0
-garak==0.10.3.1
-google-api-core==2.24.2
-google-api-python-client==2.168.0
-google-auth==2.39.0
-google-auth-httplib2==0.2.0
-googleapis-common-protos==1.70.0
-greenlet==3.2.1
-h11==0.14.0
-hf-xet==1.1.1
-httpcore==1.0.8
-httplib2==0.22.0
-httptools==0.6.4
-httpx==0.28.1
-httpx-sse==0.4.0
-huggingface-hub==0.31.2
-humanfriendly==10.0
-idna==3.10
-importlib-metadata==6.11.0
-inquirerpy==0.3.4
-itsdangerous==2.2.0
-Jinja2==3.1.6
-jiter==0.9.0
-jmespath==1.0.1
-joblib==1.4.2
-jsonpatch==1.33
-jsonpath-ng==1.7.0
-jsonpointer==3.0.0
-jsonschema==4.23.0
-jsonschema-specifications==2025.4.1
-langchain==0.3.25
-langchain-community==0.3.24
-langchain-core==0.3.59
-langchain-huggingface==0.2.0
-langchain-text-splitters==0.3.8
-langsmith==0.3.33
-latex2mathml==3.77.0
-litellm==1.67.2
-lorem==0.1.1
-Markdown==3.8
-markdown-it-py==3.0.0
-markdown2==2.5.3
-MarkupSafe==3.0.2
-marshmallow==3.26.1
-mdurl==0.1.2
-mpmath==1.3.0
-multidict==6.4.3
-multiprocess==0.70.15
-mypy_extensions==1.1.0
-nemollm==0.3.5
-networkx==3.4.2
-nh3==0.2.21
-nltk==3.9.1
-numpy==1.26.4
-nvdlib==0.8.0
-nvidia-cublas-cu12==12.6.4.1
-nvidia-cuda-cupti-cu12==12.6.80
-nvidia-cuda-nvrtc-cu12==12.6.77
-nvidia-cuda-runtime-cu12==12.6.77
-nvidia-cudnn-cu12==9.5.1.17
-nvidia-cufft-cu12==11.3.0.4
-nvidia-cufile-cu12==1.11.1.6
-nvidia-curand-cu12==10.3.7.77
-nvidia-cusolver-cu12==11.7.1.2
-nvidia-cusparse-cu12==12.5.4.2
-nvidia-cusparselt-cu12==0.6.3
-nvidia-nccl-cu12==2.26.2
-nvidia-nvjitlink-cu12==12.6.85
-nvidia-nvtx-cu12==12.6.77
-octoai-sdk==0.10.1
-ollama==0.4.8
-onnx==1.18.0
-onnxruntime==1.21.0
-onnxruntime-genai==0.7.0
-openai==1.76.0
-optimum==1.25.0
-orjson==3.10.16
-packaging==24.2
-pandas==2.2.3
-pfzy==0.3.4
-pillow==10.4.0
-ply==3.11
-prompt_toolkit==3.0.50
-propcache==0.3.1
-proto-plus==1.26.1
-protobuf==6.30.2
-psutil==7.0.0
-pyarrow==19.0.1
-pyarrow-hotfix==0.6
-pyasn1==0.6.1
-pyasn1_modules==0.4.2
-pycparser==2.22
-pydantic==2.11.3
-pydantic-settings==2.9.1
-pydantic_core==2.33.1
-Pygments==2.19.1
-pyparsing==3.2.3
-pyperclip==1.9.0
-python-dateutil==2.9.0.post0
-python-dotenv==1.1.0
-python-magic==0.4.27
-python-multipart==0.0.20
-pytz==2025.2
-PyYAML==6.0.2
-RapidFuzz==3.13.0
-referencing==0.36.2
-regex==2024.11.6
-replicate==1.0.4
-requests==2.32.3
-requests-futures==1.0.2
-requests-toolbelt==1.0.0
-rich==14.0.0
-rpds-py==0.24.0
-rsa==4.9.1
-s3transfer==0.12.0
-safetensors==0.5.3
-scikit-learn==1.6.1
-scipy==1.15.3
-sentence-transformers==4.1.0
-sentencepiece==0.2.0
-setuptools==79.0.1
-shortuuid==1.0.13
-six==1.17.0
-sniffio==1.3.1
-soundfile==0.13.1
-SQLAlchemy==2.0.40
-starlette==0.46.2
-stdlibs==2025.4.4
-svgwrite==1.4.3
-sympy==1.13.3
-tenacity==9.1.2
-threadpoolctl==3.6.0
-tiktoken==0.9.0
-timm==1.0.15
-tokenizers==0.21.1
-tomli==2.2.1
-torch==2.7.0
-torchvision==0.22.0
-tqdm==4.67.1
-transformers==4.51.3
-triton==3.3.0
-types-PyYAML==6.0.12.20250402
-types-requests==2.32.0.20250328
-typing-inspect==0.9.0
-typing-inspection==0.4.0
-typing_extensions==4.13.1
-tzdata==2025.2
-uritemplate==4.1.1
-urllib3==2.3.0
-uvicorn==0.34.2
-uvloop==0.21.0
-waitress==3.0.2
-watchfiles==1.0.5
-wavedrom==2.0.3.post3
-wcwidth==0.2.13
-websockets==15.0.1
-Werkzeug==3.1.3
-wn==0.9.5
-xdg-base-dirs==6.0.2
-xxhash==3.5.0
-yarl==1.20.0
-zalgolib==0.2.2
-zipp==3.21.0
-zope.interface==7.2
-zstandard==0.23.0

src/llm/llm.py

@@ -47,7 +47,7 @@ class Phi3LanguageModel:
             local_files_only=True
         )
         model = ORTModelForCausalLM.from_pretrained(
-            model_path,  # Change model_id to just model_path
+            model_path, 
             provider="CPUExecutionProvider",
             trust_remote_code=True,
             local_files_only=True
@@ -59,7 +59,7 @@ class Phi3LanguageModel:
             "text-generation",
             model=model,
             tokenizer=tokenizer,
-            max_new_tokens=512,
+            max_new_tokens=256,
             temperature=0.7,
             top_p=0.9,
             repetition_penalty=1.1,