diff --git a/src/Makefile.am b/src/Makefile.am
index 65c9d1d..0dfcd35 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -13,6 +13,7 @@ libapfl_a_SOURCES += expr.c
 libapfl_a_SOURCES += format.c
 libapfl_a_SOURCES += gc.c
 libapfl_a_SOURCES += hashmap.c
+libapfl_a_SOURCES += messages.c
 libapfl_a_SOURCES += parser.c
 libapfl_a_SOURCES += position.c
 libapfl_a_SOURCES += resizable.c
diff --git a/src/apfl.h b/src/apfl.h
index 74595f0..7b74310 100644
--- a/src/apfl.h
+++ b/src/apfl.h
@@ -194,6 +194,9 @@ struct apfl_error {
 bool apfl_error_print(struct apfl_error, FILE *);
 bool apfl_error_as_string(struct apfl_error, struct apfl_allocator, struct apfl_string *out);
 
+// Get a constant string for the error, if available (returns NULL otherwise)
+const char *apfl_error_as_const_string(struct apfl_error error);
+
 struct apfl_error apfl_error_simple(enum apfl_error_type);
 bool apfl_error_is_fatal_type(enum apfl_error_type);
 
@@ -626,6 +629,14 @@ enum apfl_result apfl_get_member(apfl_ctx, apfl_stackidx container, apfl_stackid
 
 bool apfl_debug_print_val(apfl_ctx, apfl_stackidx, FILE *);
 
+struct apfl_messages {
+    const char *could_not_alloc_mem;
+    const char *input_error_while_parsing;
+    const char *unexpected_end_of_file;
+    const char *feature_not_implemented;
+};
+extern const struct apfl_messages apfl_messages;
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/error.c b/src/error.c
index 9a6582e..468433b 100644
--- a/src/error.c
+++ b/src/error.c
@@ -65,16 +65,33 @@ apfl_error_type_name(enum apfl_error_type type)
     return "<unknown error>";
 }
 
+const char *
+apfl_error_as_const_string(struct apfl_error error)
+{
+    switch (error.type) {
+    case APFL_ERR_MALLOC_FAILED:
+        return apfl_messages.could_not_alloc_mem;
+    case APFL_ERR_INPUT_ERROR:
+        return apfl_messages.input_error_while_parsing;
+    case APFL_ERR_UNEXPECTED_EOF:
+        return apfl_messages.unexpected_end_of_file;
+    case APFL_ERR_NOT_IMPLEMENTED:
+        return apfl_messages.feature_not_implemented;
+    default:
+        return NULL;
+    }
+}
+
 static bool
 format_error(struct apfl_format_writer w, struct apfl_error error)
 {
     switch (error.type) {
     case APFL_ERR_MALLOC_FAILED:
-        return apfl_format_put_string(w, "Could not allocate memory");
+        return apfl_format_put_string(w, apfl_messages.could_not_alloc_mem);
     case APFL_ERR_INPUT_ERROR:
-        return apfl_format_put_string(w, "Input error while parsing");
+        return apfl_format_put_string(w, apfl_messages.input_error_while_parsing);
     case APFL_ERR_UNEXPECTED_EOF:
-        return apfl_format_put_string(w, "Unexpected end of file");
+        return apfl_format_put_string(w, apfl_messages.unexpected_end_of_file);
     case APFL_ERR_EXPECTED_EQ_AFTER_COLON:
         TRY(apfl_format_put_string(w, "Expected '=' after ':' at "));
         TRY(apfl_format_put_pos(w, error.position));
@@ -170,7 +187,7 @@ format_error(struct apfl_format_writer w, struct apfl_error error)
         TRY(apfl_format_put_pos(w, error.position));
         return true;
     case APFL_ERR_NOT_IMPLEMENTED:
-        TRY(apfl_format_put_string(w, "Feature not implemented"));
+        TRY(apfl_format_put_string(w, apfl_messages.feature_not_implemented));
         return true;
     }
 
diff --git a/src/eval.c b/src/eval.c
index c59aea3..8444088 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -232,6 +232,23 @@ apfl_iterative_runner_new(apfl_ctx ctx, struct apfl_source_reader reader)
     return runner;
 }
 
+static bool
+move_string_onto_stack(apfl_ctx ctx, struct apfl_string string)
+{
+    struct apfl_value *value = apfl_stack_push_placeholder(ctx);
+    if (value == NULL) {
+        return false;
+    }
+
+    if ((value->string = apfl_string_move_into_new_gc_string(&ctx->gc, &string)) == NULL) {
+        return false;
+    }
+
+    value->type = VALUE_STRING;
+
+    return true;
+}
+
 static enum apfl_result
 handle_parse_error(apfl_ctx ctx, struct apfl_error error)
 {
@@ -240,7 +257,12 @@ handle_parse_error(apfl_ctx ctx, struct apfl_error error)
         result = APFL_RESULT_ERR_FATAL;
     }
 
-    // TODO: Putting the string on the stack should be easier. It's rather awkward now...
+    const char *const_str = apfl_error_as_const_string(error);
+    if (const_str != NULL) {
+        return apfl_push_const_string(ctx, const_str)
+            ? result
+            : APFL_RESULT_ERR_FATAL;
+    }
 
     struct apfl_string string;
     if (!apfl_error_as_string(error, ctx->gc.allocator, &string)) {
@@ -248,17 +270,10 @@ handle_parse_error(apfl_ctx ctx, struct apfl_error error)
         return APFL_RESULT_ERR_FATAL;
     }
 
-    struct apfl_value *value = apfl_stack_push_placeholder(ctx);
-    if (value == NULL) {
+    if (!move_string_onto_stack(ctx, string)) {
         return APFL_RESULT_ERR_FATAL;
     }
 
-    if ((value->string = apfl_string_move_into_new_gc_string(&ctx->gc, &string)) == NULL) {
-        return APFL_RESULT_ERR_FATAL;
-    }
-
-    value->type = VALUE_STRING;
-
     return result;
 }
 
diff --git a/src/messages.c b/src/messages.c
new file mode 100644
index 0000000..6a4e366
--- /dev/null
+++ b/src/messages.c
@@ -0,0 +1,8 @@
+#include "apfl.h"
+
+const struct apfl_messages apfl_messages = {
+    .could_not_alloc_mem = "Could not allocate memory",
+    .input_error_while_parsing = "Input error while parsing",
+    .unexpected_end_of_file = "Unexpected end of file",
+    .feature_not_implemented = "Feature not implemented",
+};