1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
| --------------------------------------------------------------------------------------------------
-- This protected type handles the tokenisation of strings. Treat it as an object instantiation.
--
-- This protected type takes in a string an breaks it into separate strings using the given
-- delimiter, or it reaches the end of the string, at which point it will return a null string.
-- Below is an example of it's use showing all containing functions
--
-- eg.
-- variable st : string_tokeniser_t;
-- variable my_str : line; --Line should be used because you dont know the string
-- --length until it is returned
--
-- st.setstring("Hello|World");
-- st.set_delim("|");
--
-- echo(st.get_next_token) --puts "Hello" onto console
-- echo(st.get_next_token) --puts "World" onto console
--
-- st.restart;
--
-- for i in 1 to st.count_tokens('|') loop
-- echo(st.get_next_token); -- puts "Hello" on colsole when i = 1,
-- -- "World" when i = 2
-- end loop;
--
-- st.cleanup --to ensure all pointers are deleted
--
-- Common Parameters
--
-- delim : The character(s) to use as string delimiters.
-- S : The string to be tokenised
-- rtn_null : If true, delimiters are absolute. ie. 2 adjacent delimiters would mean return a
-- a string of 0 length.
-- When false, if the return string length is 0, keep trying until a non-null string
-- is found. (default to false)
--------------------------------------------------------------------------------------------------
type string_tokeniser_t is protected
---------------------------------------------------------------------
--Set the string that needs tokenising
---------------------------------------------------------------------
procedure set_string(s : string);
-------------------------------------------------------------------------------------------
--set the delimiter to use during tokenising and counting tokens
-- if a string is set, all characters in the string are considered delimeters
------------------------------------------------------------------------------------------
procedure set_delim(delim : character);
procedure set_delim(delim : string);
procedure set_delim_whitespace;
-------------------------------------------------------------------------------------------
-- With this set false, two adjacent delimeters will cause a null token to be returned
-- Set true, all delimeters are skipped between tokens (unless end of string is reached)
-------------------------------------------------------------------------------------------
procedure set_return_null_tokens(rtn_null : boolean);
---------------------------------------------------------------------
-- When set true, the delimeter is also returned (default false)
---------------------------------------------------------------------
procedure set_return_delim( rtn : boolean );
---------------------------------------------------------------------
--restart the tokenising process with the previously entered string
---------------------------------------------------------------------
procedure restart;
---------------------------------------------------------------------
--DEALLOCATE all pointers (may help prevent memory leaks)
---------------------------------------------------------------------
procedure cleanup;
---------------------------------------------------------------------
--Get the next token from the string specified
--If no delimiter specified, the last used delimiter will be used
---------------------------------------------------------------------
impure function get_next_token(delim : character;
rtn_null : boolean := false) return string;
impure function get_next_token(delim : string;
rtn_null : boolean := false) return string;
impure function get_next_token return string;
---------------------------------------------------------------------
-- Same as the get_next_token functions, but return nothing
-- used as a means to take tokens off the string
---------------------------------------------------------------------
procedure bin_next_token(delim : character;
rtn_null : boolean := false);
procedure bin_next_token(delim : string;
rtn_null : boolean := false);
procedure bin_next_token;
---------------------------------------------------------------------
--The number of tokens remaining in the current string
---------------------------------------------------------------------
impure function count_tokens(delim : character;
rtn_null : boolean := false) return natural;
impure function count_tokens(delim : string;
rtn_null : boolean := false) return natural;
impure function count_tokens return natural;
end protected string_tokeniser_t;
type string_tokeniser_t is protected body
variable tok_str : line;
variable next_tok_idx : natural;
variable d : line;
variable rtn_n : boolean := false;
variable r_delim : boolean := false;
--------------------------------------------------
--Set procedures for setting internal variables
--------------------------------------------------
procedure set_string( s : string) is
begin
--clean up existing instance
DEALLOCATE(tok_str);
--Open a new memory area and put the data into it
tok_str := new string(s'range);
tok_str.all := s;
next_tok_idx := tok_str.all'low; --starting point of the string
end procedure;
procedure set_delim(delim : character) is
begin
DEALLOCATE(d);
d := new string'(delim & "");
end procedure set_delim;
procedure set_delim(delim : string) is
begin
DEALLOCATE(d);
d := new string'(delim);
end procedure set_delim;
procedure set_delim_whitespace is
begin
DEALLOCATE(d);
d := new string'(" " & HT & LF & VT & FF & CR );
end procedure set_delim_whitespace;
--------------------------------------------------------------------------------------------
-- With this set false, two adjacent delimeters will cause a null token to be returned
-- Set true, all delimeters are skipped between tokens (unless end of string is reached)
--------------------------------------------------------------------------------------------
procedure set_return_null_tokens(rtn_null : boolean) is
begin
rtn_n := rtn_null;
end procedure set_return_null_tokens;
procedure set_return_delim(rtn : boolean) is
begin
r_delim := rtn;
end procedure set_return_delim;
--------------------------------------------------------
--Restart tokenising from the start of the string
--------------------------------------------------------
procedure restart is
begin
next_tok_idx := tok_str.all'low;
end procedure restart;
procedure cleanup is
begin
DEALLOCATE(tok_str);
DEALLOCATE(d);
end procedure cleanup;
--------------------------------------------------------------------------
-- match the character c to any of the chars in the delim string
--------------------------------------------------------------------------
impure function match_delim( c : character) return boolean is
begin
for i in d.all'range loop
if c = d.all(i) then return true;
end if;
end loop;
return false;
end function match_delim;
------------------------------------------------------------------
--Get the next token from the string bounded by the delimeter
------------------------------------------------------------------
impure function get_next_token return string is
variable temp : natural;
begin
if tok_str = null then
report "string_tokeniser-get_next_token: Null pointer found - returning null string"
severity WARNING;
return "";
elsif next_tok_idx > tok_str.all'high then --Previously ended a string
--no need to report. String exists, but all tokens returned
return "";
else
--User wants to search the previous specified string
search_loop_old : for i in next_tok_idx to tok_str.all'high loop
if match_delim(tok_str.all(i)) then --found delimiter match
temp := next_tok_idx;
next_tok_idx := i + 1; --this is where the next search should start
-------------------------------
-- 0 length between delims
-------------------------------
if next_tok_idx - temp = 1 then
------------------------------------
-- user wants a null string
------------------------------------
if rtn_n then
if r_delim then
return tok_str.all(next_tok_idx-1 to next_tok_idx-1);
else
return "";
end if;
end if;
else
if r_delim then
return tok_str.all(temp to i);
else
return tok_str.all(temp to i-1);
end if;
end if;
-- If rtn_n = false this should keep looping until it's found a new
-- retrunable token, stripping out delimeters
end if;
end loop search_loop_old;
--run off the end of the string, so return from the last know start index to the end
temp := next_tok_idx;
next_tok_idx := tok_str.all'high + 1; --This is off the end so subsequent
--calls return null
return tok_str.all(temp to tok_str.all'high);
end if;
end function get_next_token;
impure function get_next_token(delim : character;
rtn_null : boolean := false) return string is
begin
DEALLOCATE(d);
d := new string'(delim & "");
rtn_n := rtn_null;
return get_next_token;
end function get_next_token;
impure function get_next_token(delim : string;
rtn_null : boolean := false) return string is
begin
DEALLOCATE(d);
d := new string'(delim);
rtn_n := rtn_null;
return get_next_token;
end function get_next_token;
------------------------------------------------------------------
--Do the same as the function above, but return nothing.
------------------------------------------------------------------
procedure bin_next_token is
variable temp : natural;
begin
if tok_str = null then
report "string_tokeniser-get_next_token: Null pointer found (No string set for tokenising)"
severity WARNING;
elsif next_tok_idx > tok_str.all'high then --Previously ended a string
--no need to report. String exists, but all tokens returned
null;
else
--User wants to search the previous specified string
search_loop_old : for i in next_tok_idx to tok_str.all'high loop
if match_delim(tok_str.all(i)) then --found delimiter match
temp := next_tok_idx;
next_tok_idx := i + 1; --this is where the next search should start
-------------------------------
-- 0 length between delims
-------------------------------
if next_tok_idx - temp = 1 then
-----------------------------------------
-- user interested in 0 length strings
-----------------------------------------
if rtn_n then
return;
end if;
else
return;
end if;
-- If rtn_n = false this should keep looping until it's found a new
-- retrunable token, stripping out delimeters
end if;
end loop search_loop_old;
--run off the end of the string, so return from the last know start index to the end
temp := next_tok_idx;
next_tok_idx := tok_str.all'high + 1; --This is off the end so subsequent
--calls return null
end if;
end procedure bin_next_token;
procedure bin_next_token(delim : character;
rtn_null : boolean := false) is
begin
DEALLOCATE(d);
d := new string'(delim & "");
rtn_n := rtn_null;
bin_next_token;
end procedure bin_next_token;
procedure bin_next_token(delim : string;
rtn_null : boolean := false) is
begin
DEALLOCATE(d);
d := new string'(delim);
rtn_n := rtn_null;
bin_next_token;
end procedure bin_next_token;
-----------------------------------------------------------------------------
--count the number of tokens left in the string bounded by the delimiter
-----------------------------------------------------------------------------
impure function count_tokens return natural is
variable n_tok : natural := 0;
variable prev_delim_i : natural;
begin
n_tok := 0;
if tok_str = null then
report "string_tokeniser-count_tokens: Null pointer detected, no string available to count."
severity WARNING;
else
--count the number of tokens left in the current stored string
for i in next_tok_idx to tok_str.all'high loop
if match_delim(tok_str.all(i)) --found a match
or i = tok_str.all'high --end of the string
then
if i = prev_delim_i + 1 then -- null token found
if rtn_n then
n_tok := n_tok + 1;
end if;
else
n_tok := n_tok + 1;
end if;
prev_delim_i := i; -- store the index of this delimeter
end if;
end loop;
end if;
return n_tok;
end function count_tokens;
impure function count_tokens(delim : character;
rtn_null : boolean := false) return natural is
begin
DEALLOCATE(d);
d := new string'(delim & "");
rtn_n := rtn_null;
return count_tokens;
end function count_tokens;
impure function count_tokens(delim : string;
rtn_null : boolean := false) return natural is
begin
DEALLOCATE(d);
d := new string'(delim);
rtn_n := rtn_null;
return count_tokens;
end function count_tokens;
end protected body string_tokeniser_t; |