model generation not working

[LucienShui]

huggingface-vscode-endpoint-server/main.py

Lines 18 to 42 in 91848af

	pretrained = "bigcode/starcoder"
	tokenizer = AutoTokenizer.from_pretrained(pretrained, trust_remote_code=True)
	model = AutoModelForCausalLM.from_pretrained(pretrained, trust_remote_code=True)
	pipe: Pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, device_map="auto")
	app = FastAPI()
	logger = logging.getLogger(__name__)
	app.add_middleware(
	CORSMiddleware
	)
	@app.post("/api/generate/")
	async def chat(request: Request):
	json_request: dict = await request.json()
	inputs: str = json_request['inputs']
	parameters: dict = json_request['parameters']
	config: GenerationConfig = GenerationConfig(**parameters)
	logger.info(f'{request.client.host}:{request.client.port} inputs = {inputs}')
	generated_text: str = pipe(inputs, generation_config=config)
	return {
	"generated_text": generated_text,
	"status": 200
	}

File "/home/debian/.local/miniconda3/envs/all-in-one/lib/python3.10/site-packages/transformers/pipelines/text_generation.py", line 251, in _forward
    generated_sequence = self.model.generate(input_ids=input_ids, attention_mask=attention_mask, **generate_kwargs)
  File "/home/debian/.local/miniconda3/envs/all-in-one/lib/python3.10/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
    return func(*args, **kwargs)
  File "/home/debian/.local/miniconda3/envs/all-in-one/lib/python3.10/site-packages/transformers/generation/utils.py", line 1485, in generate
    return self.sample(
  File "/home/debian/.local/miniconda3/envs/all-in-one/lib/python3.10/site-packages/transformers/generation/utils.py", line 2565, in sample
    raise ValueError("If `eos_token_id` is defined, make sure that `pad_token_id` is defined.")
ValueError: If `eos_token_id` is defined, make sure that `pad_token_id` is defined.