Production QLoRA Pipeline

            bnb_config = BitsAndBytesConfig(
                load_in_4bit=True,
                bnb_4bit_quant_type="nf4",
                bnb_4bit_compute_dtype=torch_dtype, # Use the determined torch_dtype here
                bnb_4bit_use_double_quant=True,
            )
            logger.info("BitsAndBytes 4-bit quantization enabled.")
        ...
        try:
            model = AutoModelForCausalLM.from_pretrained(
                model_name,
                quantization_config=bnb_config,
                device_map="auto",
                trust_remote_code=True,
                torch_dtype=torch_dtype,
                low_cpu_mem_usage=True
            )
            logger.info(f"Model {model_name} loaded successfully.")

            # CRITICAL: Prepare model for k-bit training BEFORE applying LoRA
            if bnb_config is not None:
                logger.info("Preparing model for k-bit training...")
                model = prepare_model_for_kbit_training(model)
                logger.info("Model prepared for k-bit training.")

            # Moved to TrainingArguments for SFTTrainer
            # model.gradient_checkpointing_enable()
            model.config.use_cache = False # Important for training

            # Ensure tokenizer settings are reflected in model config for compatibility
            if tokenizer.pad_token_id is not None:
                model.config.pad_token_id = tokenizer.pad_token_id
            if tokenizer.bos_token_id is not None:
                model.config.bos_token_id = tokenizer.bos_token_id
            if tokenizer.eos_token_id is not None:
                model.config.eos_token_id = tokenizer.eos_token_id

        lora_config = LoraConfig(
            r=self.config["lora_r"],
            lora_alpha=self.config["lora_alpha"],
            target_modules=target_modules,
            lora_dropout=self.config["lora_dropout"],
            bias="none",
            task_type=TaskType.CAUSAL_LM,
        )
        model = get_peft_model(model, lora_config)

        callbacks = []
        if val_dataset:
            early_stopping = EarlyStoppingCallback(
                early_stopping_patience=self.config.get("early_stopping_patience", 3),
                early_stopping_threshold=self.config.get("early_stopping_threshold", 0.001),
            )
            callbacks.append(early_stopping)
        if self.is_cuda_available:
            callbacks.append(self.EmptyCacheCallback())
            callbacks.append(self.MemoryMonitoringCallback())

        trainer = SFTTrainer(
            model=model,
            train_dataset=train_dataset,     # list of {"messages": [...]} — trl 0.12 applies the chat template
            eval_dataset=val_dataset,
            args=training_args,
            callbacks=callbacks,
            tokenizer=tokenizer,
            max_seq_length=self.config.get("max_seq_length", 1024),
        )
        train_result = trainer.train()
        trainer.model.save_pretrained(str(output_dir / "final_adapter"))
        tokenizer.save_pretrained(str(output_dir / "tokenizer"))

def load_generation_model(base_model_name: str, adapter_path: str):
    """Loads the fine-tuned Gemma-9B model for text generation."""
    if not os.path.exists(adapter_path):
        raise FileNotFoundError(f"Adapter path '{adapter_path}' not found.")

    bnb_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_use_double_quant=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=torch.float16,
    )

    base_model = AutoModelForCausalLM.from_pretrained(
        base_model_name,
        quantization_config=bnb_config,
        device_map="auto",
        trust_remote_code=True
    )
    tokenizer = AutoTokenizer.from_pretrained(adapter_path, trust_remote_code=True)
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token

    model = PeftModel.from_pretrained(base_model, adapter_path)
    print("Merging adapters...")
    model = model.merge_and_unload()
    model.eval()
    return model, tokenizer

from langchain import PromptTemplate, LLMChain
from llama_wrapper import LlamaWrapper
from langchain.prompts import PromptTemplate
from langchain.chains import LLMChain
import re

def generate_summary(text):
    print("Generate summary started \n")
    """Generates a summary of the input text using Langchain and Hugging Face."""
    llm = LlamaWrapper(
        api_host="http://<internal-vllm-host>:8111",   # real host redacted for publication
        api_key="<redacted>",                        # hardcoded in the original; redacted here
        model="meta-llama/Llama-3.3-70B-Instruct",
        temperature=0.5,
        max_tokens=500
    )
    template = """Write a concise summary of the following text:
    Text: {text}
    Summary:"""
    prompt = PromptTemplate(template=template, input_variables=["text"])
    summary_chain = prompt | llm
    return summary_chain.invoke({"text": text})

        template = """
        Generate a clear and structured instructional guide on the topic below.
        The instruction must:
        - Provide a step-by-step breakdown
        - Be easy to follow and precise
        - Avoid repetition and unnecessary details
        - Use bullet points or numbering for clarity

        **Topic:** {prompt_text}
        **Number of Steps:** {num_steps}

        **Instructional Guide:**
        """

        prompt = PromptTemplate(template=template, input_variables=["prompt_text", "num_steps"])
        instruction_chain = prompt | llm

        raw_output = instruction_chain.invoke({"prompt_text": prompt_text, "num_steps": num_steps}).strip()

        clean_instruction = re.sub(r'[^A-Za-z0-9.,\'"!?;:\s]', '', raw_output)  # Remove random symbols
        clean_instruction = re.sub(r'\b(\w+)\s+\1\b', r'\1', clean_instruction)  # Remove repeated words

        # **Tagging the Output**
        tagged_instruction = f"[Official Commercient Content]\n\n{clean_instruction}"

        return tagged_instruction

model_name = "meta-llama/Meta-Llama-3.1-8B"
tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token  # Set padding token

# Tokenization function (Set up labels for loss calculation)
def tokenize_function(examples):
    # Check if the input is "instruction" and "output" or just "text"
    if "instruction" in examples and "output" in examples:
        # Combine instruction and output for training
        texts = [f"{instruction}\n{output}" for instruction, output in zip(examples["instruction"], examples["output"])]
    elif "text" in examples:
        texts = examples["text"]
    else:
        raise ValueError("Dataset must contain either 'instruction' and 'output' keys, or a 'text' key.")

    tokenized = tokenizer(texts, padding="max_length", truncation=True, max_length=512)
    tokenized["labels"] = tokenized["input_ids"].copy()  # Labels needed for loss computation
    return tokenized

dataset = Dataset.from_list(dataset)
tokenized_dataset = dataset.map(tokenize_function, batched=True)

# Load LLaMA 3.1 model in 4-bit
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_use_double_quant=True,
    bnb_4bit_compute_dtype=torch.float16
)

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    quantization_config=bnb_config,
    device_map="auto"
)
print(model_name)
# Apply QLoRA
lora_config = LoraConfig(
    r=8,  # LoRA rank
    lora_alpha=32,
    target_modules=["q_proj", "v_proj"],
    lora_dropout=0.1,
    bias="none",
    task_type="CAUSAL_LM",
    base_model_name_or_path=model_name  # Fix PEFT warning
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()

# Use Data Collator to avoid unnecessary padding
data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)

# Training Arguments (2 GPUs)
training_args = TrainingArguments(
    output_dir="./fine_tunned_llama3_1_qlora_ansh_v_6_TAG_JSON",
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    evaluation_strategy="no",
    save_strategy="epoch",
    save_total_limit=2,
    logging_steps=10,
    learning_rate=2e-4,
    weight_decay=0.01,
    num_train_epochs=3,
    bf16=True,  # Use bfloat16
    fp16=False,
    optim="paged_adamw_32bit",
    ddp_find_unused_parameters=False,
    report_to="none"
)

# Trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_dataset,
    data_collator=data_collator
)

trainer.train()

model.save_pretrained("./fine_tunned_llama3_1_qlora_ansh_v_6_TAG_JSON")
tokenizer.save_pretrained("./fine_tunned_llama3_1_qlora_ansh_v_6_TAG_JSON")

# Define fine-tuned model path
fine_tuned_model_path = "fine_tuned_llama3_qlora_ansh_v"

# Load tokenizer
tokenizer = AutoTokenizer.from_pretrained(fine_tuned_model_path)
tokenizer.pad_token = tokenizer.eos_token  # Ensure proper padding

# Load the model in 4-bit precision using bitsandbytes
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_use_double_quant=True,
    bnb_4bit_compute_dtype=torch.float16
)

# Load base model with quantization
base_model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8b",  # meta-llama/Meta-Llama-3-8B
    quantization_config=bnb_config,
    device_map="auto"  # Automatically distributes across multiple GPUs
)

# Load fine-tuned QLoRA adapters
model = PeftModel.from_pretrained(base_model, fine_tuned_model_path)
model.eval()

print("✅ Fine-tuned model loaded successfully across multiple GPUs!")

embedding_model_name = "sentence-transformers/all-mpnet-base-v2"  # Choose a suitable model
embedding_model = AutoModel.from_pretrained(embedding_model_name).to("cuda") #move embedding model to gpu
embedding_tokenizer = AutoTokenizer.from_pretrained(embedding_model_name)


def get_embedding(text):
    """
    Generates an embedding for the given text using the Sentence Transformer model.
    """
    input_ids = embedding_tokenizer(text, return_tensors="pt", truncation=True, padding=True).to("cuda") #move input to gpu
    with torch.no_grad():
        outputs = embedding_model(**input_ids)
        embeddings = outputs.last_hidden_state.mean(dim=1)  # Mean pooling
    return embeddings.cpu().numpy() #move embedding to cpu


source_document = "Commercient integrates CRM and ERP systems, allowing businesses to streamline operations and enhance efficiency."
source_embedding = get_embedding(source_document)


def generate_response(prompt, similarity_threshold=0.5):
    """
    Generates a response based on the prompt.  Calculates the cosine similarity
    between the prompt and the source document.
    """
    prompt_embedding = get_embedding(prompt)
    similarity_score = cosine_similarity(prompt_embedding, source_embedding)[0][0]
    print(similarity_score)
    if similarity_score >= similarity_threshold:
        inputs = tokenizer(prompt, return_tensors="pt").to("cuda")  # Move to GPU
        with torch.no_grad():
            output = model.generate(**inputs, max_new_tokens=200)
        return tokenizer.decode(output[0], skip_special_tokens=True)
    else:
        return "I am designed to answer questions specifically related to Commercient's integration of CRM and ERP systems.  Your query falls outside of that scope."

def clean_text(text):
    return re.sub(r"[^a-zA-Z0-9\s:]", "", text)

with open('Filtered_Synthetic_data.txt','w') as f:
    sentences = nltk.sent_tokenize(str(results))      # str() of a Python list: "\n" becomes the two characters \ and n
    for i in sentences:
        f.write(f"{clean_text(i)} \n")                # clean_text strips the backslash and leaves a stray "n"

def parse_data(filepath, commercient_tag="[COM_SPECIAL]"):
    data = []
    with open(filepath, 'r', encoding='utf-8') as f:
        text = f.read()

    def tag_content(content, is_commercient=True):
        return f"{commercient_tag} {content}" if is_commercient else content

    # 1. Parse Q&A Pairs   (note the "\nnA" — the regex was written to match the corrupted text)
    qa_pairs = re.findall(r"Q(\d+): (.*?)\nnA(\d+): (.*?)(?=\nQ|$)", text, re.DOTALL)

    for q_num, question, a_num, answer in qa_pairs:
        is_commercient = "Commercient" in question
        data.append({
            "instruction": tag_content(question.strip(), is_commercient),
            "output": tag_content(answer.strip(), is_commercient),
            "tag": "official commercient content" if is_commercient else "general"
        })

template = """You are an expert at generating diverse and insightful Frequently Asked Questions (FAQs) from a given document.  Your goal is to create {num_pairs} FAQ pairs that cover a wide range of topics, including common questions, edge cases, and questions that require critical thinking.  Focus on generating creative and thought-provoking questions, and provide *detailed and comprehensive* answers based on the document.  Avoid simple, obvious questions.  Prioritize diversity and depth in the questions and answers.  Pay special attention to potential misunderstandings or areas where clarification would be beneficial. Aim for answers that are multiple sentences long and provide thorough explanations.

Here are some guidelines:
*   **Diversity:** Cover various aspects of the topic discussed in the document. Consider different user perspectives and potential use cases.
*   **Creativity:**  Go beyond the surface level.  Formulate questions that require deeper understanding and synthesis of information. Think about 'what if' scenarios and potential problems.
*   **Clarity:** Ensure both the question and answer are clear, concise, and easy to understand.  Avoid jargon unless necessary, and if used, explain it.
*   **Accuracy:** All answers must be factually correct and based on the provided document. If the document does not provide an answer, state that the answer is not available in the provided document.
*   **Format:** Present each FAQ pair in the following format:
    Question: [The question]
    Answer: [The detailed answer]

Document: {text}

FAQs:"""

model = prepare_model_for_kbit_training(model)

# Configure LoRA
config = LoraConfig(
    r=LORA_RANK,
    lora_alpha=2 * LORA_RANK,
    lora_dropout=0.2,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=[
        "q_proj",
        "k_proj",
        "v_proj",
        "o_proj",
        "gate_proj",
        "up_proj",
        "down_proj",
        "linear"
    ]  # Adjust target modules based on Llama 3 architecture if needed
)
model = get_peft_model(model, config)
model.print_trainable_parameters()  # Print the number of trainable parameters

# ----------------------- Data Collator -----------------------

data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=False, pad_to_multiple_of=8, return_tensors="pt")

def is_related_to_data(answer, keywords):
    answer = answer.lower()
    for keyword in keywords:
        if keyword.lower() in answer:
            return True
    return False

def generate_answer_with_rejection(model, tokenizer, question, keywords, max_attempts=3):
    for _ in range(max_attempts):
        answer = generate_answer(model, tokenizer, question)  # Use the prompt-engineered function

        if "i am sorry" in answer.lower():  # Check for the "out of scope" phrase
            return "I am sorry, I am not trained to answer question outside of my knowledge domain"

        if is_related_to_data(answer, keywords):
            return answer

    return "I am sorry, I am not trained to answer question outside of my knowledge domain"  # Default if all attempts fail

    llm = ChatGroq(
        model_name="llama-3.3-70b-versatile",  # Specify the Groq model
        temperature=0.6,  # Increased temperature for more creativity
        max_tokens=2048,   # Increased max_tokens to allow for longer, more detailed answers
        groq_api_key=GROQ_API_KEY # Use the API key from .env
    )

def create_json_from_faq(faq_text, tag="official commercient content"):
    faq_pairs = []
    pairs = faq_text.split("Question:")  # Split the text by question to extract the content
    for pair in pairs[1:]:  # Skip the first empty element
        try:
            question, answer = pair.split("Answer:")
            faq_pairs.append({
                "question": question.strip(),
                "input": "Official Commercient Content",
                "output": answer.strip()
            })
        except ValueError as e:
            print(f"Error parsing FAQ pair: {e}, Pair content: {pair}")  # Debugging print
            continue

    return json.dumps(faq_pairs, indent=4)

max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name = "unsloth/Meta-Llama-3.1-8B",
    max_seq_length = max_seq_length,
    dtype = dtype,
    load_in_4bit = load_in_4bit,
)

model = FastLanguageModel.get_peft_model(
    model,
    r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
    target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
                      "gate_proj", "up_proj", "down_proj",],
    lora_alpha = 16,
    lora_dropout = 0, # Supports any, but = 0 is optimized
    bias = "none",    # Supports any, but = "none" is optimized
    # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
    use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
    random_state = 3407,
    use_rslora = False,  # We support rank stabilized LoRA
    loftq_config = None, # And LoftQ
)

alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.

### Instruction:
{}

### Input:
{}

### Response:
{}"""

EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN

def formatting_prompts_func(examples):
    instructions = examples["instruction"]
    inputs       = examples["input"]
    outputs      = examples["output"]
    texts = []
    for instruction, input, output in zip(instructions, inputs, outputs):
        # Must add EOS_TOKEN, otherwise your generation will go on forever!
        text = alpaca_prompt.format(instruction, input, output) + EOS_TOKEN
        texts.append(text)
    return { "text" : texts, }

dataset = load_dataset("json", data_files="/home/<user>/Ansh/Unsloth/Unsolth_commercient_finetune.json", split="train")
dataset = dataset.map(formatting_prompts_func, batched = True,)

trainer = SFTTrainer(
    model = model,
    tokenizer = tokenizer,
    train_dataset = dataset,
    dataset_text_field = "text",
    max_seq_length = 512,  # Lower for VRAM savings
    dataset_num_proc = 1,  # Stick to fewer processes
    packing = True,
    args = TrainingArguments(
        per_device_train_batch_size = 2,  # Small batch size to avoid OOM
        gradient_accumulation_steps = 10,  # Accumulate gradients to compensate
        warmup_steps = 10,  # Gradual warmup
        max_steps = 40,  # Reasonable training time for your setup
        learning_rate = 2e-4,  # Learning rate for stability
        fp16 = False,  # Nvidia struggles with FP16, so leave it off
        bf16 = is_bfloat16_supported(),
        logging_steps = 1,
        optim = "adamw_8bit",  # Memory-efficient optimizer
        weight_decay = 0.01,
        lr_scheduler_type = "cosine",  # Cosine decay is usually stable
        seed = 3407,
        output_dir = "outputs",
        report_to = "none",
        gradient_checkpointing = True,  # Keep VRAM usage low
    ),
)
trainer_stats = trainer.train()
print(trainer_stats)
model.save_pretrained("Lamma_3_1_8B_Qlora_Co_Unsolth_Version_1_1")  # Local saving
tokenizer.save_pretrained("Lamma_3_1_8B_Qlora_Co_Unsolth_Version_1_1")

==((====))==  Unsloth - 2x faster free finetuning | Num GPUs used = 1
   \\   /|    Num examples = 48 | Num Epochs = 30 | Total steps = 60
O^O/ \_/ \    Batch size per device = 2 | Gradient accumulation steps = 12
\        /    Data Parallel GPUs = 1 | Total batch size (2 x 12 x 1) = 24
 "-____-"     Trainable parameters = 41,943,040/4,670,623,744 (0.90% trained)

def preprocess_data(examples):
    prompt = examples["instruction"]
    output = examples["output"]
    input = examples['input']

    # Format the input
    input_text = f"### Instruction:\n{prompt}\n\n### input:\n{input}\n\n### output:\n{output}"

    # ✅ FIXED: Proper Tokenization with Padding
    tokenized = tokenizer(
        input_text,
        padding="max_length",
        truncation=True,
        max_length=768,  # Increased max length for better context
        return_tensors="pt" # Add return_tensors="pt" to return PyTorch tensors
    )

    # Set labels (model needs input_ids for training)
    tokenized["labels"] = tokenized["input_ids"].squeeze()
    tokenized["input_ids"] = tokenized["input_ids"].squeeze() # Squeeze input_ids as well
    tokenized["attention_mask"] = tokenized["attention_mask"].squeeze() # Squeeze attention_mask as well

    return tokenized

# ✅ LoRA Configuration (Low-Rank Adaptation for Efficiency)
lora_config = LoraConfig(
    r=12,  # Increased LoRA Rank for better adaptation
    lora_alpha=32,  # More impact on fine-tuning
    lora_dropout=0.1,  # Increased dropout to prevent overfitting
    bias="none",
    task_type="CAUSAL_LM"
)
model = get_peft_model(model, lora_config)

model_name = "meta-llama/Meta-Llama-3.1-8B"  # Or your base model name
adapter_path = "/home/<user>/Ansh/Torchtune/fine_tuned_llama_torchtune_V_2"  # Path to your fine-tuned LoRA adapter

tokenizer = AutoTokenizer.from_pretrained(adapter_path)  # Load from adapter path!
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    device_map="auto",  # Or specify your device
    torch_dtype=torch.bfloat16,
)
model = PeftModel.from_pretrained(model, adapter_path)  # Load LoRA adapter

def generate_prompt(instruction, input_text="Official Commercient Content"):
    prompt = f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request, using ONLY information related to Commercient. If you do not have any answer then say 'I am unable to provide response for this query'.

### Instruction:
{instruction}

### Input:
{input_text}

### Response:
"""
    return prompt

def generate_faqs_from_text_file(data):

    try:
        text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=200)
        chunks = text_splitter.split_text(data)

        groq_llm = ChatGroq(model_name="llama-3.3-70b-versatile", groq_api_key=GROQ_API_KEY) # Using Llama 3 70B versatile model
        output_parser = JsonOutputParser()
        # 4. Define Prompt for FAQ generation in Alpaca format
        template = """You are an expert in generating Frequently Asked Questions (FAQs) based on given text content.
        Your task is to generate around 50 FAQ pairs for each chunk of text provided.
        Each FAQ pair should be in the Alpaca format (instruction, input, output).
        The 'input' field should always be 'Official Commercient Content'.
        The 'instruction' field should be the question phrased as an FAQ.
        The 'output' field should be a concise and informative answer to the FAQ, derived from the provided text chunk.

        Ensure the output is a structured JSON list of dictionaries, where each dictionary represents an FAQ pair in Alpaca format.

        Format your response as JSON.
        {format_instructions}

        Text Chunk:
        {text_chunk}
        """

        prompt = ChatPromptTemplate.from_messages([
            ("system", "You are a helpful AI assistant that generates FAQs."),
            HumanMessagePromptTemplate.from_template(template)
        ])

        # Get format instructions from the output parser
        partial_prompt = prompt.partial(format_instructions=output_parser.get_format_instructions())

        # 5. Create LLMChain
        faq_chain = LLMChain(llm=groq_llm, prompt=partial_prompt, output_parser=output_parser)

                # Clean response (sometimes contains markdown code blocks)
                cleaned_response = response.strip().replace('```json', '').replace('```', '').strip()

                try:
                    episodes = json.loads(cleaned_response)

                    if not isinstance(episodes, list):
                        print(f"Warning: Expected list but got {type(episodes)}")
                        continue

                    valid_episodes = 0
                    for episode in episodes:
                        if (isinstance(episode, dict) and
                            episode.get("input") == "Official Commercient Content" and
                            len(episode.get("podcast_style", [])) >= min_conversations_per_pair):
                            all_podcast_data.append(episode)
                            valid_episodes += 1

                    print(f"Added {valid_episodes} valid episodes from this chunk")

                except json.JSONDecodeError as e:
                    print(f"JSON parsing failed: {e}")
                    print("Problematic response (first 200 chars):", cleaned_response[:200]) # Use cleaned_response here.

    def detect_format(self, example):
        """Auto-detect format based on data structure"""
        if 'podcast_style' in example:
            return 'podcast'
        elif 'story' in example and all(k in example['story'] for k in ['beginning', 'middle', 'end']):
            return 'story'
        elif 'instructions' in example and isinstance(example['instructions'], list):
            return 'instruction'
        elif ('question' in example and 'answer' in example) or ('instruction' in example and 'output' in example):
            return 'faq'
        else:
            raise ValueError(f"Cannot detect format for example: {example.keys()}")

    def format_to_text(self, example):
        """Convert any format to standardized text with markers"""
        format_type = example["format"]

        if format_type == "faq":
            question = example.get('question', example.get('instruction', ''))
            answer = example.get('answer', example.get('output', ''))
            return f"[FAQ]\nQuestion: {question}\nAnswer: {answer}"
        elif format_type == "podcast":
            return f"[PODCAST]\n" + "\n".join(
                f"{turn['speaker']}: {turn['line']}"
                for turn in example["podcast_style"]
            )
        elif format_type == "instruction":
            return f"[INSTRUCTION]\n" + "\n".join(example["instructions"])
        elif format_type == "story":
            return f"[STORY]\nBeginning: {example['story']['beginning']}\n" \
                   f"Middle: {example['story']['middle']}\n" \
                   f"End: {example['story']['end']}"

def preprocess_data(examples):
    # Extract the full text
    text = examples["text"]

    # Parse the content type, main content, and tag
    content_parts = text.split("\nTag: ")
    content_type, main_content = content_parts[0].split("]", 1)
    content_type = content_type[1:]  # Remove the leading "["
    main_content = main_content.strip()
    tag = content_parts[1].strip() if len(content_parts) > 1 else "Unknown"

    # Format based on content type
    if content_type == "FAQ":
        # Split into question and answer
        parts = main_content.split("\n")
        question = parts[0].replace("Question: ", "")
        answer = parts[1].replace("Answer: ", "")
        formatted_text = f"### Instruction:\n{question}\n\n### input:\nTag: {tag}\n\n### output:\n{answer}"

    elif content_type == "STORY":
        # Split into beginning, middle, end
        parts = main_content.split("\n")
        beginning = parts[0].replace("Beginning: ", "")
        middle = parts[1].replace("Middle: ", "")
        end = parts[2].replace("End: ", "")
        formatted_text = f"### Instruction:\nTell me a business transformation story\n\n### input:\nTag: {tag}\nBeginning: {beginning}\nMiddle: {middle}\n\n### output:\n{end}"

    elif content_type == "PODCAST":
        # Extract the dialogue
        dialogue = "\n".join([line for line in main_content.split("\n") if line])
        formatted_text = f"### Instruction:\nGenerate podcast dialogue about business systems\n\n### input:\nTag: {tag}\n\n### output:\n{dialogue}"

    elif content_type == "INSTRUCTION":
        # Use the steps as the output
        steps = "\n".join([line for line in main_content.split("\n") if line.startswith("Step")])
        formatted_text = f"### Instruction:\nProvide steps for ERP system implementation\n\n### input:\nTag: {tag}\n\n### output:\n{steps}"

search_space = {
    "batch_size": tune.choice([4, 8, 16]),
    "learning_rate": tune.loguniform(1e-5, 1e-3),
    "num_epochs": tune.choice([2, 3, 4])
}

scheduler = ASHAScheduler(
    max_t=10,
    grace_period=1,
    reduction_factor=2
)

search_algorithm = OptunaSearch()

tuner = tune.Tuner(
    trainable=train_model,
    param_space=search_space,
    tune_config=tune.TuneConfig(
        metric="val_loss",
        mode="min",
        scheduler=scheduler,
        search_alg=search_algorithm,
        num_samples=10  # Number of configurations to try
    ),
    run_config=RunConfig(
        name="finetune_ray_tune",
        storage_path=storage_path  # Updated to use storage_path
    )
)

results = tuner.fit()
best_result = results.get_best_result(metric="val_loss", mode="min")
print("Best hyperparameters: ", best_result.config)

# Data collator
data_collator = DataCollatorForSeq2Seq(tokenizer, model=model, pad_to_multiple_of=8)

# LoRA configuration
lora_config = LoraConfig(
    r=12,
    lora_alpha=32,
    lora_dropout=0.1,
    bias="none",
    task_type="CAUSAL_LM"
)
model = get_peft_model(model, lora_config)

# Training arguments
training_args = TrainingArguments(
    output_dir="./fine_tuned_model",
    per_device_train_batch_size=4,
    gradient_accumulation_steps=8,
    save_strategy="steps",
    logging_steps=4,
    learning_rate=2e-4,
    weight_decay=0.01,
    warmup_steps=10,
    lr_scheduler_type="cosine",
    max_grad_norm=0.3,
    bf16=True,
    num_train_epochs=8,
    save_total_limit=3,
    # load_best_model_at_end=True,
    remove_unused_columns=False,
    push_to_hub=False,
    report_to="wandb",
    logging_dir="./tensorboard_logs",
    optim="adamw_8bit",
    dataloader_drop_last=True,
)

# Initialize SFTTrainer
trainer = SFTTrainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=val_dataset,
    tokenizer=tokenizer,
    data_collator=data_collator,
)

# Train the model
trainer.train()

# Save the trained model
trainer.save_model("./Commercient_fine_tuned_model")
tokenizer.save_pretrained("./Commercient_fine_tuned_model")

    data = pd.read_csv(csv_file)
    if 'Questions' not in data.columns or 'is_relevant' not in data.columns:
        raise ValueError("The CSV file must contain 'Questions' and 'is_relevant' columns.")

    X = data['Questions']  # Features (questions)
    y = data['is_relevant']  # Labels (0 or 1)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    vectorizer = TfidfVectorizer(stop_words="english", max_features=5000)
    X_train_tfidf = vectorizer.fit_transform(X_train)  # Fit and transform the training data
    X_test_tfidf = vectorizer.transform(X_test)  # Transform the test data

    model = LogisticRegression(random_state=42, max_iter=1000)
    model.fit(X_train_tfidf, y_train)

    y_pred = model.predict(X_test_tfidf)
    accuracy = accuracy_score(y_test, y_pred)
    print("Classification Report:")
    print(classification_report(y_test, y_pred))
    print(f"Accuracy on the test dataset: {accuracy:.2f}")

    joblib.dump(model, model_output_path)
    joblib.dump(vectorizer, vectorizer_output_path)

        if is_podcast_query(user_query):
            response = generate_podcast_response(user_query)
        elif is_story_query(user_query):
            response = generate_story_response(user_query)
        else:
            relevance = classify_question_relevance(user_query)
            if relevance == 1:
                response = generate_commercient_response(user_query)
            else:
                response = ("Thank you for your question. However, it seems to fall outside the scope "
                            "of Commercient's domain expertise. Please feel free to ask questions "
                            "related to Commercient, CRM, or ERP systems, and I'll be happy to assist you.")

        print(f"Commercient Bot: {response}")

GEMMA_FINE_TUNING = {
    "model_name": "unsloth/gemma-2-9b-it",
    "output_dir": "models/fine_tuned_gemma_v_1",
    "lora_config": {
        "r": 16,
        "lora_alpha": 32,
        "lora_dropout": 0.1,
        "bias": "none",
        "target_modules": ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
        "use_gradient_checkpointing": True
    },
    "quantization_config": {
        "load_in_4bit": True,
        "bnb_4bit_compute_dtype": "float16",
        "bnb_4bit_quant_type": "nf4",
        "bnb_4bit_use_double_quant": True
    },
    "device_map": "auto",  #{"": 0}
    "training_args": {
        "per_device_train_batch_size": 1,
        "per_device_eval_batch_size": 1,
        "gradient_accumulation_steps": 10,
        "learning_rate": 2e-5,
        "num_train_epochs": 2,
        "eval_strategy": "steps",
        "eval_steps": 100,
        "save_strategy": "steps",
        "save_steps": 100,
        "logging_steps": 10,
        "lr_scheduler_type": "cosine",
        "save_total_limit": 3,
        "warmup_ratio": 0.1,
        "max_grad_norm": 0.3,
        "weight_decay": 0.2,
        "optim": "adamw_torch",
        "group_by_length": True,
        "metric_for_best_model": "eval_loss",
        "greater_is_better": False,
        "early_stopping_patience": 5,
        "early_stopping_threshold": 0.01
    }
}

    def _init_model(self) -> None:
        """Initialize the model with quantization and LoRA."""
        logger.info(f"Loading model: {self.model_name}")

        quantization_config = None
        if self.quantization_config:
            logger.info("Applying quantization configuration")
            quantization_config = BitsAndBytesConfig(**self.quantization_config)

        self.base_model = AutoModelForCausalLM.from_pretrained(
            self.model_name,
            quantization_config=quantization_config,
            device_map=self.device_map,
            cache_dir=self.cache_dir,
            trust_remote_code=True
        )

        if quantization_config:
            logger.info("Preparing model for k-bit training")
            self.base_model = prepare_model_for_kbit_training(
                self.base_model,
                use_gradient_checkpointing=self.lora_config.get("use_gradient_checkpointing", True)
            )

        logger.info("Applying LoRA configuration")
        peft_config = LoraConfig(
            task_type=TaskType.CAUSAL_LM,
            **{k: v for k, v in self.lora_config.items() if k != "use_gradient_checkpointing"}
        )

        self.model = get_peft_model(self.base_model, peft_config)
        self.model.print_trainable_parameters()

Training on 119665 samples with batch size 1 ... Total training steps: 23932 ... Estimated training time: 19.94 hours
ERROR - Error during training: You can't train a model that has been loaded in 8-bit precision on a
different device than the one you're training on. Make sure you loaded the model on the correct device
using for example `device_map={'':torch.cuda.current_device()}` or `device_map={'':torch.xpu.current_device()}`
Attempting to save checkpoint to models/fine_tuned_gemma_v_1/error_checkpoint

        text_splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=100)
        chunks = text_splitter.split_text(data)
        ...
        template = """You are an expert in generating Frequently Asked Questions (FAQs) based on given text content.
Your task is to generate a JSON list containing around {faqs_per_chunk} FAQ pairs based on the provided text chunk.
Each FAQ pair should be formatted as a JSON object with three keys: 'instruction', 'input', and 'output'.

Here are the specific requirements for each key:
1.  'instruction': This field should contain a clear instruction for a Commercient assistant. It must start with "You are a helpful assistant for Commercient, providing information ONLY about Commercient, CRM, and ERP." followed by the specific task, like "Answer the following question:".
2.  'input': This field should contain the actual question derived from the text chunk, phrased as a user query.
3.  'output': This field should contain a concise and informative answer to the question in the 'input' field, derived *only* from the provided text chunk.

Ensure the overall output is a structured JSON list of these {faqs_per_chunk} FAQ dictionaries.

Format your response as JSON.
{format_instructions}

Text Chunk:
{text_chunk}
"""

        # Use Langchain's prompt template. We still use JsonOutputParser to get the format instructions.
        output_parser = JsonOutputParser()

        prompt_template = ChatPromptTemplate.from_messages([
            ("system", "You are a helpful AI assistant that generates data in a specific JSON format."),
            HumanMessagePromptTemplate.from_template(template)
        ])

        cleaned_text = raw_response_text.strip()
        parsed_data = None

        # Try robustly extracting the main JSON block (expecting a list [ ... ])
        list_start = cleaned_text.find('[')
        list_end = cleaned_text.rfind(']')
        if list_start != -1 and list_end != -1 and list_end > list_start:
             potential_json_text = cleaned_text[list_start : list_end + 1]
             try:
                  parsed_data = json.loads(potential_json_text)
             except json.JSONDecodeError:
                  parsed_data = None # Reset if parsing fails

        # If main block parsing failed, try parsing the whole cleaned text (after fence removal)
        if parsed_data is None:
             if cleaned_text.startswith('```json'):
                 cleaned_text = cleaned_text[len('```json'):].strip()
             if cleaned_text.endswith('```'):
                 cleaned_text = cleaned_text[:-len('```')].strip()
             try:
                  parsed_data = json.loads(cleaned_text)
             except json.JSONDecodeError:
                  parsed_data = None

        # --- Salvage Attempt: Only if primary parsing failed or yielded very few results ---
        salvage_threshold = FAQS_PER_CHUNK * 0.95
        if len(valid_entries) < salvage_threshold or parsed_data is None:
             potential_objects = re.findall(r'\{\s*"instruction"\s*:\s*".*?".*?"input"\s*:\s*".*?".*?"output"\s*:\s*".*?".*?\}', raw_response_text, re.DOTALL)
             for obj_str in potential_objects:
                  try:
                       item = json.loads(obj_str)
                       item_tuple = tuple(sorted(item.items()))
                       if is_valid_faq_entry(item) and item_tuple not in [tuple(sorted(e.items())) for e in valid_entries]:
                            valid_entries.append(item)
                  except json.JSONDecodeError:
                       logging.warning("Failed to parse a potential salvaged object string as JSON.")

adapter: lora
base_model: meta-llama/Meta-Llama-3.1-8B-Instruct
chat_template: chatml
datasets:
- conversation: chatml
  path: data/processed/train.json
  type: sharegpt
deepspeed: configs/zero2.json
lora_r: 32
lora_alpha: 16
lora_dropout: 0.05
lora_target_linear: true
learning_rate: 0.0002
lr_scheduler: cosine
micro_batch_size: 2
gradient_accumulation_steps: 4
num_epochs: 3
optimizer: adamw_bnb_8bit
sample_packing: true
sequence_len: 2048
train_on_inputs: false        # completion-only loss — the setting every hand-rolled trainer lacked
early_stopping_patience: 3
eval_steps: 100
special_tokens:
  pad_token: <|end_of_text|>  # a real pad token distinct from EOS

        instruction = str(item.get("instruction", "")).strip()
        input_text = str(item.get("input", "")).strip()
        output_text = str(item.get("output", "")).strip()

        # Construct the first user message content
        user_content = ""
        if instruction and input_text:
            user_content = instruction + "\n\n" + input_text # Combine system and user with separator
        elif instruction:
            user_content = instruction # Use only instruction if no input
        elif input_text:
            user_content = input_text # Use only input if no instruction

        # Build the messages list
        messages = []
        if user_content:
             messages.append({"role": "user", "content": user_content})
             # Only add assistant turn if there was a preceding user turn AND assistant content exists
             if output_text:
                  messages.append({"role": "assistant", "content": output_text})
        elif output_text:
             logger.warning(f"Skipping item {i}: Only 'output' field contains content without preceding instruction/input. Content: {item}")
             skipped_count += 1
             continue

    def get_target_modules(self, model):
        """
        Dynamically get target modules based on model architecture
        """
        configured_modules = self.config.get("lora_target_modules", [])

        all_modules = set()
        for name, module in model.named_modules():
            if len(list(module.children())) == 0:  # leaf modules
                all_modules.add(name.split('.')[-1])

        logger.info(f"Available leaf modules in model: {sorted(list(all_modules))}")

        valid_modules = []
        for module_name in configured_modules:
            if any(module_name == name.split('.')[-1] for name, _ in model.named_modules() if len(list(module.children())) == 0):
                valid_modules.append(module_name)
            else:
                if module_name in ["embed_tokens", "lm_head"]:
                    logger.warning(f"Configured target module '{module_name}' not typically recommended for LoRA due to high memory consumption, skipping. Consider removing it from your config.")
                else:
                    logger.warning(f"Configured target module '{module_name}' not found in model or is not a leaf module.")

        if not valid_modules:
            logger.warning("No configured target modules found or valid. Using default attention modules.")
            common_targets = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
            ...
        if not valid_modules:
            raise ValueError("No suitable target modules found for LoRA adaptation!")

        return valid_modules

    class EmptyCacheCallback(TrainerCallback):
        def on_step_end(self, args, state, control, **kwargs):
            if state.global_step > 0 and state.global_step % 50 == 0:
                if torch.cuda.is_available():
                    torch.cuda.empty_cache()

        def on_evaluate(self, args, state, control, **kwargs):
            if torch.cuda.is_available():
                torch.cuda.empty_cache()

    class MemoryMonitoringCallback(TrainerCallback):
        def on_step_end(self, args, state, control, **kwargs):
            if state.global_step > 0 and state.global_step % 20 == 0: # Log every 20 steps
                if torch.cuda.is_available():
                    allocated = torch.cuda.memory_allocated() / (1024**3)
                    cached = torch.cuda.memory_reserved() / (1024**3)
                    logger.info(f"Step {state.global_step} - GPU Memory: Allocated: {allocated:.2f} GB, Cached: {cached:.2f} GB")

        except Exception as e:
            logger.error(f"An error occurred during trainer.train(): {e}", exc_info=True)
            try:
                error_checkpoint_path = output_dir / f"error_checkpoint_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
                trainer.save_model(str(error_checkpoint_path))
                logger.warning(f"Saved a checkpoint to {error_checkpoint_path} after encountering an error during training.")
            except Exception as save_e:
                logger.error(f"Failed to save checkpoint after error during training: {save_e}", exc_info=True)
            raise e

def get_available_gpus(min_memory_gb=12, max_gpus=2):
    """
    Returns a list of GPU indices that have at least `min_memory_gb` free memory,
    up to a maximum of `max_gpus`.
    """
    pynvml.nvmlInit()
    gpu_count = pynvml.nvmlDeviceGetCount()
    available_gpus = []

    for i in range(gpu_count):
        handle = pynvml.nvmlDeviceGetHandleByIndex(i)
        memory_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
        free_memory_gb = memory_info.free / (1024 ** 3)
        print(f"GPU {i}: Free memory = {free_memory_gb:.2f} GB")

        if free_memory_gb >= min_memory_gb:
            available_gpus.append(i)
            if len(available_gpus) == max_gpus:
                break # Exit the loop early

    pynvml.nvmlShutdown()
    return available_gpus


class CommercientFineTuner:
    def __init__(self, config_path, output_path=None, source_path=None):
        # Dynamic GPU selection
        available_gpus = get_available_gpus(min_memory_gb=18)
        if available_gpus:
            os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, available_gpus))
            logger.info(f"Using GPUs: {os.environ['CUDA_VISIBLE_DEVICES']}")
        else:
            log_to_customer_json("ERROR:- Fine tuning process Failed( No Sufficent Memory found in GPUs )")
            logger.error("No GPU with sufficient memory found! Stopping execution.")
            os._exit(1) # Forceful exit

    def chunk_text_by_tokens(self, text: str, max_tokens: int = None) -> List[str]:
        """Split text into chunks based on token count."""
        if max_tokens is None:
            max_tokens = self.token_limit

        token_count = self.count_tokens(text)
        if token_count <= max_tokens:
            return [text]

        char_limit = max_tokens * 4
        overlap_chars = 200

        text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=char_limit,
            chunk_overlap=overlap_chars,
            length_function=len,
            separators=["\n\n", "\n", ". ", " ", ""]
        )

        chunks = text_splitter.split_text(text)
        verified_chunks = []

        for chunk in chunks:
            chunk_tokens = self.count_tokens(chunk)
            if chunk_tokens <= max_tokens:
                verified_chunks.append(chunk)
            else:
                logging.warning(f"Chunk still too large ({chunk_tokens} tokens), splitting further")
                smaller_char_limit = char_limit // 2
                smaller_splitter = RecursiveCharacterTextSplitter(
                    chunk_size=smaller_char_limit,
                    chunk_overlap=overlap_chars // 2,
                    length_function=len
                )
                verified_chunks.extend(smaller_splitter.split_text(chunk))

        return verified_chunks

    base_template = """You are an expert in generating Frequently Asked Questions (FAQs) based on given text content.
Your task is to generate a JSON list containing approximately {faqs_per_row} FAQ pairs based on the provided text content.
Focus on generating questions and answers that are directly answerable *only* from the text provided.

Each FAQ pair should be formatted as a JSON object with three keys: 'instruction', 'input', and 'output'.

Here are the specific requirements for each key:
1.  'instruction': This field must contain a clear instruction for a Commercient assistant. It must start with "You are a helpful assistant for Commercient, providing information ONLY about Commercient, CRM, and ERP." followed by the specific task, like "Answer the following question:". Ensure this prefix is exactly as specified.
2.  'input': This field should contain the actual question derived from the text content, phrased as a user query.
3.  'output': This field should contain a concise and informative answer to the question in the 'input' field, derived *only* from the provided text content. Do not include information not present in the text.

{custom_prompt_addition}

Ensure the overall output is a structured JSON list of approximately {faqs_per_row} FAQ dictionaries.

Format your response as JSON.
{format_instructions}

Source Text Content:
{source_text}
"""

    # Combine base template with customer customization
    full_template = base_template.replace(
        "{custom_prompt_addition}",
        prompt_customization if prompt_customization else ""
    )

@app.post("/generate", response_model=GenerationResponse)
async def generate_response_api(request: GenerationRequest):
    user_query = request.prompt

    # 1. Retrieve relevant context from Pinecone
    try:
        retrieved_results = query_all_namespaces(
            index=pinecone_index,
            query_text=user_query,
            total_top_k=5,
            fetch_k_per_namespace=10
        )

        # Simple relevance check
        if not retrieved_results or retrieved_results[0]['score'] < 0.4:
            print("Context relevance below threshold. Replying with a standard message.")
            return GenerationResponse(response_text="I can only answer questions related to Commercient. How can I help you with that?")

    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error retrieving context from Pinecone: {e}")

    try:
        messages = [
            {"role": "user", "content": f"\n\nQuestion: {request.prompt}"}
        ]

        prompt_formatted = tokenizer.apply_chat_template(
            messages,
            tokenize=False,
            add_generation_prompt=True
        )

        inputs = tokenizer(prompt_formatted, return_tensors="pt").to(generation_model.device)

        with torch.no_grad():
            outputs = generation_model.generate(
                **inputs,
                max_new_tokens=request.max_new_tokens,
                temperature=request.temperature,
                top_p=request.top_p,
                do_sample=True,
                eos_token_id=tokenizer.eos_token_id,
                pad_token_id=tokenizer.eos_token_id
            )

        response_text = tokenizer.decode(outputs[0][len(inputs["input_ids"][0]):], skip_special_tokens=True)
        return GenerationResponse(response_text=response_text.strip())

    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

{"instruction": "You are a helpful assistant for Commercient, providing information ONLY about Commercient, CRM, and ERP. Answer the following question:",
 "input": "What server access is required to configure the ODBC DSN for Commercient?",
 "output": "You must have access to the server where your ERP is installed."}

{"instruction": "You are a helpful assistant for Commercient, providing information ONLY about Commercient, CRM, and ERP. Provide step-by-step instructions for accessing the CRM App in Commercient Admin.",
 "input": "Process: Accessing the CRM App in Commercient Admin",
 "output": "Step 1: Navigate to the Commercient Admin screen.\nStep 2: Go to the CRM setup section.\nStep 3: ..."}

{"instruction": "You are a helpful assistant for Commercient, providing information ONLY about Commercient, CRM, and ERP. Generate a short podcast-style conversation about [Topic].",
 "input": "Topic: CRM Setup Location in Admin Screen",
 "output": "Host: Where can I access the CRM setup in Commercient? Guest: In the Commercient Admin screen, navigate to the CRM setup section. There, you'll find a direct link to the CRM App."}

{"instruction": "You are a helpful assistant for Commercient, providing information ONLY about Commercient, CRM, and ERP. Tell a very detailed and engaging story about how a retail company overcame a data synchronization bottleneck using the Commercient CRM Sync App.",
 "input": "Company/User Type: Retail Startup, Challenge: Inability to sync more than 10 CRM records with their ERP, limiting customer outreach, Solution: Upgraded Commercient license enabling full Sync App access.",
 "output": "At BrightBoutique, a rapidly growing online retailer, the team faced a critical roadblock. ..."}

# Alpaca: a fixed English scaffold, model-agnostic, needs an explicit EOS
alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.

### Instruction:
{}

### Input:
{}

### Response:
{}"""
text = alpaca_prompt.format(instruction, input, output) + tokenizer.eos_token

# Chat template: the model's own control tokens, rendered by the tokenizer
messages = [
    {"role": "user", "content": instruction + "\n\n" + input},   # Gemma-IT has no system role
    {"role": "assistant", "content": output},
]
text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
# → "<bos><start_of_turn>user\n…<end_of_turn>\n<start_of_turn>model\n…<end_of_turn>\n"