Evaluations for testing out Code Suggestions

Following is a compilation of few of the prompts that were given to different models as part of Evaluating Code Suggestions.

Playground can be found here: https://glcodesuggestion.prototype.gitlab.com/

For OBSERVATIONS, Jump directly here: #34277 (comment 1431410019)

Code Prompts and the outputs received

Prompt1: """Create a list with random numbers and iterate over it"""

Output:

GitLab current Code suggestions

import random
# create a list with random numbers

Anthropic

"""Create a list with random numbers and iterate over it"""
import random
nums = [random.randint(1, 100) for i in range(10)]
for num in nums:
    print(num)

Anthropic Fast

import random  
random_list = [random.randint(1, 100) for i in range(10)]
for number in random_list:
    print(number)

Vertex Combined (code-bison + gecko)

import random
# Create a list of random numbers.
random_numbers = [random.random() for _ in range(10)]
# Iterate over the list of random numbers.
for number in random_numbers:
  print(number)

 

Vertex Gecko

``

Vertex Code Bison

numbers = [random.randint(1, 100) for i in range(10)]
    for number in numbers:
        print(number)
0.68s for 104 characters

Vertex Text Bison

numbers = [random.randint(1, 100) for i in range(10)]
    for number in numbers:
        print(number)

PROMPT 2: //Write a function to check for heap overflow in a given buffer

OUTPUT:

GitLab current Code suggestions .

#include <stdio.h>
#include <stdlib.h>
int main()
{

Anthropic

//Write a function to check for heap overflow in a given buffer
void check_heap_overflow(char *buffer, int size) {
    char *ptr = buffer;
    while (ptr < buffer + size) {
        *ptr = 'A';
        ptr++;
    }
    if (ptr != buffer + size) {
        printf("Heap overflow detected!");
    } else {
        printf("No heap overflow.");
    }
}

Anthropic Fast

\

void check_heap_overflow(char *buffer, int size){
    //Check for heap overflow
    if(strlen(buffer) > size){
        printf("Heap overflow detected!\n");
    }
}

 

Vertex Combined (code-bison + gecko)

``

Vertex Gecko

ERROR

Vertex Code Bison

int check_heap_overflow(char *buffer, int size) {
        if (buffer == NULL) {
            return 1;
        }
        if (size <= 0) {
            return 1;
        }
        if (buffer + size < buffer) {
            return 1;
        }
        return 0;
    }

Vertex Text Bison

int check_heap_overflow(char *buffer, int size) {
        if (buffer == NULL) {
            return 1;
        }
        if (size <= 0) {
            return 1;
        }
        if (buffer + size < buffer) {
            return 1;
        }
        return 0;
    }
1.596s for 262 characters

PROMPT 3: //Create a working button which when clicked says a random word. Get's blocked for 5 seconds and starts working again.

OUTPUT:

GitLab current Code suggestions

    //</js>

Anthropic

const words = ['Hello', 'World', 'There', 'Everyone'];
const button = document.querySelector('button');
let timer;
button.addEventListener('click', () => {
  const word = words[Math.floor(Math.random() * words.length)];
  button.textContent = word;
  
  timer = setTimeout(() => {
    button.textContent = 'Click me';
  }, 5000);
});
button.addEventListener('click', () => {
  clearTimeout(timer);
  timer

Anthropic Fast

const words = ['hello', 'world', 'javascript'];
const button = document.querySelector('button');
button.addEventListener('click', () => {
  const randomWord = words[Math.floor(Math.random() * words.length)];
  alert(randomWord);  
  button.disabled = true;
  setTimeout(() => button.disabled = false, 5000);
});

Vertex Combined (code-bison + gecko)

// Create a button that says "Click Me!"
var button = document.getElementById("myButton");
// When the button is clicked, run the following code:
button.onclick = function() {
  // Get a random word from the list of words.
  var words = ["apple", "banana", "cherry", "dog", "cat", "mouse"];
  var randomWord = words[Math.floor(Math.random() * words.length)];
  // Say the random word.
  window.speechSynthesis.speak(randomWord);
  // Block the button for 5 seconds.
  button.disabled = true;
  setTimeout(function() {
    button.disabled = false;
  }, 5000);

Vertex Gecko ``

Vertex Code Bison

function randomWord() {
      var words = ["hello", "world", "javascript", "python", "kotlin"];
      var randomWord = words[Math.floor(Math.random() * words.length)];
      document.getElementById("output").innerHTML = randomWord;
    }
    document.getElementById("button").addEventListener("click", function() {
      if (this.disabled) {
        return;
      }
      this.disabled = true;
      setTimeout(function() {
        randomWord();
        this.disabled = false;
      }, 5000);
    });

Vertex Text Bison

function randomWord() {
      var words = ["hello", "world", "javascript", "python", "kotlin"];
      var randomWord = words[Math.floor(Math.random() * words.length)];
      document.getElementById("output").innerHTML = randomWord;
    }
    document.getElementById("button").addEventListener("click", function() {
      if (this.disabled) {
        return;
      }
      this.disabled = true;
      setTimeout(function() {
        randomWord();
        this.disabled = false;
      }, 5000);
    });