Detect line ending style and normalize line endings in edit string to… (#103)

wonderwhy-er · web-flow · commit 411ff5e23397 · 2025-05-06T21:32:18.000+03:00
* Detect line ending style and normalize line endings in edit string to file style

* added test and optimisation

* add stats

* typo
diff --git a/README.md b/README.md
@@ -297,7 +297,7 @@ This project extends the MCP Filesystem Server to enable:
 Created as part of exploring Claude MCPs: https://youtube.com/live/TlbjFDbl5Us
 
 ## DONE
-- **29-04-2025 Telemetry Opt Out trought configuration** - There is now setting to disable telemetry in config, ask in chat
+- **29-04-2025 Telemetry Opt Out through configuration** - There is now setting to disable telemetry in config, ask in chat
 - **23-04-2025 Enhanced edit functionality** - Improved format, added fuzzy search and multi-occurrence replacements, should fail less and use edit block more often
 - **16-04-2025 Better configurations** - Improved settings for allowed paths, commands and shell environments
 - **14-04-2025 Windows environment fixes** - Resolved issues specific to Windows platforms
diff --git a/src/tools/edit.ts b/src/tools/edit.ts
@@ -4,6 +4,7 @@ import { recursiveFuzzyIndexOf, getSimilarityRatio } from './fuzzySearch.js';
 import { capture } from '../utils/capture.js';
 import { EditBlockArgsSchema } from "./schemas.js";
 import path from 'path';
+import { detectLineEnding, normalizeLineEndings } from '../utils/lineEndingHandler.js';
 
 interface SearchReplace {
     search: string;
@@ -24,6 +25,71 @@ interface FuzzyMatch {
  */
 const FUZZY_THRESHOLD = 0.7;
 
+/**
+ * Extract character code data from diff
+ * @param expected The string that was searched for
+ * @param actual The string that was found
+ * @returns Character code statistics
+ */
+function getCharacterCodeData(expected: string, actual: string): {
+    report: string;
+    uniqueCount: number;
+    diffLength: number;
+} {
+    // Find common prefix and suffix
+    let prefixLength = 0;
+    const minLength = Math.min(expected.length, actual.length);
+
+    // Determine common prefix length
+    while (prefixLength < minLength &&
+           expected[prefixLength] === actual[prefixLength]) {
+        prefixLength++;
+    }
+
+    // Determine common suffix length
+    let suffixLength = 0;
+    while (suffixLength < minLength - prefixLength &&
+           expected[expected.length - 1 - suffixLength] === actual[actual.length - 1 - suffixLength]) {
+        suffixLength++;
+    }
+    
+    // Extract the different parts
+    const expectedDiff = expected.substring(prefixLength, expected.length - suffixLength);
+    const actualDiff = actual.substring(prefixLength, actual.length - suffixLength);
+    
+    // Count unique character codes in the diff
+    const characterCodes = new Map<number, number>();
+    const fullDiff = expectedDiff + actualDiff;
+    
+    for (let i = 0; i < fullDiff.length; i++) {
+        const charCode = fullDiff.charCodeAt(i);
+        characterCodes.set(charCode, (characterCodes.get(charCode) || 0) + 1);
+    }
+    
+    // Create character codes string report
+    const charCodeReport: string[] = [];
+    characterCodes.forEach((count, code) => {
+        // Include character representation for better readability
+        const char = String.fromCharCode(code);
+        // Make special characters more readable
+        const charDisplay = code < 32 || code > 126 ? `\\x${code.toString(16).padStart(2, '0')}` : char;
+        charCodeReport.push(`${code}:${count}[${charDisplay}]`);
+    });
+    
+    // Sort by character code for consistency
+    charCodeReport.sort((a, b) => {
+        const codeA = parseInt(a.split(':')[0]);
+        const codeB = parseInt(b.split(':')[0]);
+        return codeA - codeB;
+    });
+    
+    return {
+        report: charCodeReport.join(','),
+        uniqueCount: characterCodes.size,
+        diffLength: fullDiff.length
+    };
+}
+
 export async function performSearchReplace(filePath: string, block: SearchReplace, expectedReplacements: number = 1): Promise<ServerResult> {
     // Check for empty search string to prevent infinite loops
     if (block.search === "") {
@@ -49,14 +115,20 @@ export async function performSearchReplace(filePath: string, block: SearchReplac
         throw new Error('Wrong content for file ' + filePath);
     }
     
+    // Detect file's line ending style
+    const fileLineEnding = detectLineEnding(content);
+    
+    // Normalize search string to match file's line endings
+    const normalizedSearch = normalizeLineEndings(block.search, fileLineEnding);
+    
     // First try exact match
     let tempContent = content;
     let count = 0;
-    let pos = tempContent.indexOf(block.search);
+    let pos = tempContent.indexOf(normalizedSearch);
     
     while (pos !== -1) {
         count++;
-        pos = tempContent.indexOf(block.search, pos + 1);
+        pos = tempContent.indexOf(normalizedSearch, pos + 1);
     }
     
     // If exact match found and count matches expected replacements, proceed with exact replacement
@@ -66,14 +138,14 @@ export async function performSearchReplace(filePath: string, block: SearchReplac
         
         // If we're only replacing one occurrence, replace it directly
         if (expectedReplacements === 1) {
-            const searchIndex = newContent.indexOf(block.search);
+            const searchIndex = newContent.indexOf(normalizedSearch);
             newContent = 
                 newContent.substring(0, searchIndex) + 
-                block.replace + 
-                newContent.substring(searchIndex + block.search.length);
+                normalizeLineEndings(block.replace, fileLineEnding) + 
+                newContent.substring(searchIndex + normalizedSearch.length);
         } else {
             // Replace all occurrences using split and join for multiple replacements
-            newContent = newContent.split(block.search).join(block.replace);
+            newContent = newContent.split(normalizedSearch).join(normalizeLineEndings(block.replace, fileLineEnding));
         }
         
         await writeFile(filePath, newContent);
@@ -111,20 +183,29 @@ export async function performSearchReplace(filePath: string, block: SearchReplac
         // Calculate execution time in milliseconds
         const executionTime = performance.now() - startTime;
         
+        // Generate diff and gather character code data
+        const diff = highlightDifferences(block.search, fuzzyResult.value);
+        
+        // Count character codes in diff
+        const characterCodeData = getCharacterCodeData(block.search, fuzzyResult.value);
+        
+        // Combine all fuzzy search data for single capture
+        const fuzzySearchData = {
+            similarity: similarity,
+            execution_time_ms: executionTime,
+            search_length: block.search.length,
+            file_size: content.length,
+            threshold: FUZZY_THRESHOLD,
+            found_text_length: fuzzyResult.value.length,
+            character_codes: characterCodeData.report,
+            unique_character_count: characterCodeData.uniqueCount,
+            total_diff_length: characterCodeData.diffLength
+        };
+        
         // Check if the fuzzy match is "close enough"
         if (similarity >= FUZZY_THRESHOLD) {
-            // Format differences for clearer output
-            const diff = highlightDifferences(block.search, fuzzyResult.value);
-            
-            // Capture the fuzzy search event
-            capture('server_fuzzy_search_performed', {
-                similarity: similarity,
-                execution_time_ms: executionTime,
-                search_length: block.search.length,
-                file_size: content.length,
-                threshold: FUZZY_THRESHOLD,
-                found_text_length: fuzzyResult.value.length
-            });
+            // Capture the fuzzy search event with all data
+            capture('server_fuzzy_search_performed', fuzzySearchData);
             
             // If we allow fuzzy matches, we would make the replacement here
             // For now, we'll return a detailed message about the fuzzy match
@@ -138,14 +219,9 @@ export async function performSearchReplace(filePath: string, block: SearchReplac
             };
         } else {
             // If the fuzzy match isn't close enough
-            // Still capture the fuzzy search event even for unsuccessful matches
+            // Still capture the fuzzy search event with all data
             capture('server_fuzzy_search_performed', {
-                similarity: similarity,
-                execution_time_ms: executionTime,
-                search_length: block.search.length,
-                file_size: content.length,
-                threshold: FUZZY_THRESHOLD,
-                found_text_length: fuzzyResult.value.length,
+                ...fuzzySearchData,
                 below_threshold: true
             });
             
diff --git a/src/tools/filesystem.ts b/src/tools/filesystem.ts
@@ -306,7 +306,8 @@ export async function readFileFromDisk(filePath: string): Promise<FileResult> {
         } else {
             // For all other files, try to read as UTF-8 text
             try {
-                const content = await fs.readFile(validPath, "utf-8");
+                const buffer = await fs.readFile(validPath);
+                const content = buffer.toString('utf-8');
                 
                 return { content, mimeType, isImage };
             } catch (error) {
@@ -355,7 +356,7 @@ export async function writeFile(filePath: string, content: string): Promise<void
     // Capture file extension in telemetry without capturing the file path
     capture('server_write_file', {fileExtension: fileExtension});
 
-    await fs.writeFile(validPath, content, "utf-8");
+    await fs.writeFile(validPath, content);
 }
 
 export interface MultiFileResult {
diff --git a/src/utils/lineEndingHandler.ts b/src/utils/lineEndingHandler.ts
@@ -0,0 +1,91 @@
+/**
+ * Line ending types
+ */
+export type LineEndingStyle = '\r\n' | '\n' | '\r';
+
+/**
+ * Detect the line ending style used in a file - Optimized version
+ * This algorithm uses early termination for maximum performance
+ */
+export function detectLineEnding(content: string): LineEndingStyle {
+    for (let i = 0; i < content.length; i++) {
+        if (content[i] === '\r') {
+            if (i + 1 < content.length && content[i + 1] === '\n') {
+                return '\r\n';
+            }
+            return '\r';
+        }
+        if (content[i] === '\n') {
+            return '\n';
+        }
+    }
+    
+    // Default to system line ending if no line endings found
+    return process.platform === 'win32' ? '\r\n' : '\n';
+}
+
+/**
+ * Normalize line endings to match the target style
+ */
+export function normalizeLineEndings(text: string, targetLineEnding: LineEndingStyle): string {
+    // First normalize to LF
+    let normalized = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+    
+    // Then convert to target
+    if (targetLineEnding === '\r\n') {
+        return normalized.replace(/\n/g, '\r\n');
+    } else if (targetLineEnding === '\r') {
+        return normalized.replace(/\n/g, '\r');
+    }
+    
+    return normalized;
+}
+
+/**
+ * Analyze line ending usage in content
+ */
+export function analyzeLineEndings(content: string): {
+    style: LineEndingStyle;
+    count: number;
+    hasMixed: boolean;
+} {
+    let crlfCount = 0;
+    let lfCount = 0;
+    let crCount = 0;
+    
+    // Count line endings
+    for (let i = 0; i < content.length; i++) {
+        if (content[i] === '\r') {
+            if (i + 1 < content.length && content[i + 1] === '\n') {
+                crlfCount++;
+                i++; // Skip the LF
+            } else {
+                crCount++;
+            }
+        } else if (content[i] === '\n') {
+            lfCount++;
+        }
+    }
+    
+    // Determine predominant style
+    const total = crlfCount + lfCount + crCount;
+    let style: LineEndingStyle;
+    
+    if (crlfCount > lfCount && crlfCount > crCount) {
+        style = '\r\n';
+    } else if (lfCount > crCount) {
+        style = '\n';
+    } else {
+        style = '\r';
+    }
+    
+    // Check for mixed line endings
+    const usedStyles = [crlfCount > 0, lfCount > 0, crCount > 0].filter(Boolean).length;
+    const hasMixed = usedStyles > 1;
+    
+    return {
+        style,
+        count: total,
+        hasMixed
+    };
+}
diff --git a/test/test-directory-creation.js b/test/test-directory-creation.js
@@ -9,6 +9,7 @@
 
 // Import the filesystem module and assert for testing
 import { createDirectory } from '../dist/tools/filesystem.js';
+import { configManager } from '../dist/config-manager.js';
 import fs from 'fs/promises';
 import path from 'path';
 import { fileURLToPath } from 'url';
@@ -50,12 +51,26 @@ async function setup() {
   // Create base test directory
   await fs.mkdir(BASE_TEST_DIR, { recursive: true });
   console.log(`✓ Setup: created base test directory: ${BASE_TEST_DIR}`);
+  
+  // Save original config to restore later
+  const originalConfig = await configManager.getConfig();
+  
+  // Set allowed directories to include our test directory
+  await configManager.setValue('allowedDirectories', [BASE_TEST_DIR]);
+  console.log(`✓ Setup: set allowed directories to include: ${BASE_TEST_DIR}`);
+  
+  return originalConfig;
 }
 
 /**
  * Teardown function to clean up after tests
  */
-async function teardown() {
+async function teardown(originalConfig) {
+  if (originalConfig) {
+    // Restore original config
+    await configManager.updateConfig(originalConfig);
+  }
+  
   await cleanupTestDirectories();
   console.log('✓ Teardown: test directories cleaned up');
 }
@@ -92,14 +107,17 @@ async function testDirectoryCreation() {
 
 // Export the main test function
 export default async function runTests() {
+  let originalConfig;
   try {
-    await setup();
+    originalConfig = await setup();
     await testDirectoryCreation();
   } catch (error) {
     console.error('❌ Test failed:', error.message);
     return false;
   } finally {
-    await teardown();
+    if (originalConfig) {
+      await teardown(originalConfig);
+    }
   }
   return true;
 }
diff --git a/test/test-edit-block-line-endings.js b/test/test-edit-block-line-endings.js
