1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
29 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
30 * All Rights Reserved
31 */
32
33 /*
34 * General assembly language routines.
35 * It is the intent of this file to contain routines that are
36 * independent of the specific kernel architecture, and those that are
37 * common across kernel architectures.
38 * As architectures diverge, and implementations of specific
39 * architecture-dependent routines change, the routines should be moved
40 * from this file into the respective ../`arch -k`/subr.s file.
41 */
42
43 #include <sys/asm_linkage.h>
44 #include <sys/asm_misc.h>
45 #include <sys/panic.h>
46 #include <sys/ontrap.h>
47 #include <sys/regset.h>
48 #include <sys/privregs.h>
49 #include <sys/reboot.h>
50 #include <sys/psw.h>
51 #include <sys/x86_archext.h>
52
53 #if defined(__lint)
54 #include <sys/types.h>
55 #include <sys/systm.h>
56 #include <sys/thread.h>
57 #include <sys/archsystm.h>
58 #include <sys/byteorder.h>
59 #include <sys/dtrace.h>
60 #include <sys/ftrace.h>
61 #else /* __lint */
62 #include "assym.h"
63 #endif /* __lint */
64 #include <sys/dditypes.h>
65
66 /*
67 * on_fault()
68 * Catch lofault faults. Like setjmp except it returns one
69 * if code following causes uncorrectable fault. Turned off
70 * by calling no_fault().
71 */
72
73 #if defined(__lint)
74
75 /* ARGSUSED */
76 int
77 on_fault(label_t *ljb)
78 { return (0); }
79
80 void
81 no_fault(void)
82 {}
83
84 #else /* __lint */
85
86 #if defined(__amd64)
87
88 ENTRY(on_fault)
89 movq %gs:CPU_THREAD, %rsi
90 leaq catch_fault(%rip), %rdx
91 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
92 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
93 jmp setjmp /* let setjmp do the rest */
94
95 catch_fault:
96 movq %gs:CPU_THREAD, %rsi
97 movq T_ONFAULT(%rsi), %rdi /* address of save area */
98 xorl %eax, %eax
99 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
100 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
101 jmp longjmp /* let longjmp do the rest */
102 SET_SIZE(on_fault)
103
104 ENTRY(no_fault)
105 movq %gs:CPU_THREAD, %rsi
106 xorl %eax, %eax
107 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
108 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
109 ret
110 SET_SIZE(no_fault)
111
112 #elif defined(__i386)
113
114 ENTRY(on_fault)
115 movl %gs:CPU_THREAD, %edx
116 movl 4(%esp), %eax /* jumpbuf address */
117 leal catch_fault, %ecx
118 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
119 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
120 jmp setjmp /* let setjmp do the rest */
121
122 catch_fault:
123 movl %gs:CPU_THREAD, %edx
124 xorl %eax, %eax
125 movl T_ONFAULT(%edx), %ecx /* address of save area */
126 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
127 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
128 pushl %ecx
129 call longjmp /* let longjmp do the rest */
130 SET_SIZE(on_fault)
131
132 ENTRY(no_fault)
133 movl %gs:CPU_THREAD, %edx
134 xorl %eax, %eax
135 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
136 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
137 ret
138 SET_SIZE(no_fault)
139
140 #endif /* __i386 */
141 #endif /* __lint */
142
143 /*
144 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
145 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
146 */
147
148 #if defined(lint)
149
150 void
151 on_trap_trampoline(void)
152 {}
153
154 #else /* __lint */
155
156 #if defined(__amd64)
157
158 ENTRY(on_trap_trampoline)
159 movq %gs:CPU_THREAD, %rsi
160 movq T_ONTRAP(%rsi), %rdi
161 addq $OT_JMPBUF, %rdi
162 jmp longjmp
163 SET_SIZE(on_trap_trampoline)
164
165 #elif defined(__i386)
166
167 ENTRY(on_trap_trampoline)
168 movl %gs:CPU_THREAD, %eax
169 movl T_ONTRAP(%eax), %eax
170 addl $OT_JMPBUF, %eax
171 pushl %eax
172 call longjmp
173 SET_SIZE(on_trap_trampoline)
174
175 #endif /* __i386 */
176 #endif /* __lint */
177
178 /*
179 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
180 * more information about the on_trap() mechanism. If the on_trap_data is the
181 * same as the topmost stack element, we just modify that element.
182 */
183 #if defined(lint)
184
185 /*ARGSUSED*/
186 int
187 on_trap(on_trap_data_t *otp, uint_t prot)
188 { return (0); }
189
190 #else /* __lint */
191
192 #if defined(__amd64)
193
194 ENTRY(on_trap)
195 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
196 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
197 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
198 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
199 xorl %ecx, %ecx
200 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
201 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
202 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
203 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
204 cmpq %rdi, %rcx /* if (otp == %rcx) */
205 je 0f /* don't modify t_ontrap */
206
207 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
208 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
209
210 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
211 jmp setjmp
212 SET_SIZE(on_trap)
213
214 #elif defined(__i386)
215
216 ENTRY(on_trap)
217 movl 4(%esp), %eax /* %eax = otp */
218 movl 8(%esp), %edx /* %edx = prot */
219
220 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
221 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
222 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
223 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
224 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
225 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
226 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
227 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
228 cmpl %eax, %ecx /* if (otp == %ecx) */
229 je 0f /* don't modify t_ontrap */
230
231 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
232 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
233
234 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
235 movl %eax, 4(%esp) /* put %eax back on the stack */
236 jmp setjmp /* let setjmp do the rest */
237 SET_SIZE(on_trap)
238
239 #endif /* __i386 */
240 #endif /* __lint */
241
242 /*
243 * Setjmp and longjmp implement non-local gotos using state vectors
244 * type label_t.
245 */
246
247 #if defined(__lint)
248
249 /* ARGSUSED */
250 int
251 setjmp(label_t *lp)
252 { return (0); }
253
254 /* ARGSUSED */
255 void
256 longjmp(label_t *lp)
257 {}
258
259 #else /* __lint */
260
261 #if LABEL_PC != 0
262 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
263 #endif /* LABEL_PC != 0 */
264
265 #if defined(__amd64)
266
267 ENTRY(setjmp)
268 movq %rsp, LABEL_SP(%rdi)
269 movq %rbp, LABEL_RBP(%rdi)
270 movq %rbx, LABEL_RBX(%rdi)
271 movq %r12, LABEL_R12(%rdi)
272 movq %r13, LABEL_R13(%rdi)
273 movq %r14, LABEL_R14(%rdi)
274 movq %r15, LABEL_R15(%rdi)
275 movq (%rsp), %rdx /* return address */
276 movq %rdx, (%rdi) /* LABEL_PC is 0 */
277 xorl %eax, %eax /* return 0 */
278 ret
279 SET_SIZE(setjmp)
280
281 ENTRY(longjmp)
282 movq LABEL_SP(%rdi), %rsp
283 movq LABEL_RBP(%rdi), %rbp
284 movq LABEL_RBX(%rdi), %rbx
285 movq LABEL_R12(%rdi), %r12
286 movq LABEL_R13(%rdi), %r13
287 movq LABEL_R14(%rdi), %r14
288 movq LABEL_R15(%rdi), %r15
289 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
290 movq %rdx, (%rsp)
291 xorl %eax, %eax
292 incl %eax /* return 1 */
293 ret
294 SET_SIZE(longjmp)
295
296 #elif defined(__i386)
297
298 ENTRY(setjmp)
299 movl 4(%esp), %edx /* address of save area */
300 movl %ebp, LABEL_EBP(%edx)
301 movl %ebx, LABEL_EBX(%edx)
302 movl %esi, LABEL_ESI(%edx)
303 movl %edi, LABEL_EDI(%edx)
304 movl %esp, 4(%edx)
305 movl (%esp), %ecx /* %eip (return address) */
306 movl %ecx, (%edx) /* LABEL_PC is 0 */
307 subl %eax, %eax /* return 0 */
308 ret
309 SET_SIZE(setjmp)
310
311 ENTRY(longjmp)
312 movl 4(%esp), %edx /* address of save area */
313 movl LABEL_EBP(%edx), %ebp
314 movl LABEL_EBX(%edx), %ebx
315 movl LABEL_ESI(%edx), %esi
316 movl LABEL_EDI(%edx), %edi
317 movl 4(%edx), %esp
318 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
319 movl $1, %eax
320 addl $4, %esp /* pop ret adr */
321 jmp *%ecx /* indirect */
322 SET_SIZE(longjmp)
323
324 #endif /* __i386 */
325 #endif /* __lint */
326
327 /*
328 * if a() calls b() calls caller(),
329 * caller() returns return address in a().
330 * (Note: We assume a() and b() are C routines which do the normal entry/exit
331 * sequence.)
332 */
333
334 #if defined(__lint)
335
336 caddr_t
337 caller(void)
338 { return (0); }
339
340 #else /* __lint */
341
342 #if defined(__amd64)
343
344 ENTRY(caller)
345 movq 8(%rbp), %rax /* b()'s return pc, in a() */
346 ret
347 SET_SIZE(caller)
348
349 #elif defined(__i386)
350
351 ENTRY(caller)
352 movl 4(%ebp), %eax /* b()'s return pc, in a() */
353 ret
354 SET_SIZE(caller)
355
356 #endif /* __i386 */
357 #endif /* __lint */
358
359 /*
360 * if a() calls callee(), callee() returns the
361 * return address in a();
362 */
363
364 #if defined(__lint)
365
366 caddr_t
367 callee(void)
368 { return (0); }
369
370 #else /* __lint */
371
372 #if defined(__amd64)
373
374 ENTRY(callee)
375 movq (%rsp), %rax /* callee()'s return pc, in a() */
376 ret
377 SET_SIZE(callee)
378
379 #elif defined(__i386)
380
381 ENTRY(callee)
382 movl (%esp), %eax /* callee()'s return pc, in a() */
383 ret
384 SET_SIZE(callee)
385
386 #endif /* __i386 */
387 #endif /* __lint */
388
389 /*
390 * return the current frame pointer
391 */
392
393 #if defined(__lint)
394
395 greg_t
396 getfp(void)
397 { return (0); }
398
399 #else /* __lint */
400
401 #if defined(__amd64)
402
403 ENTRY(getfp)
404 movq %rbp, %rax
405 ret
406 SET_SIZE(getfp)
407
408 #elif defined(__i386)
409
410 ENTRY(getfp)
411 movl %ebp, %eax
412 ret
413 SET_SIZE(getfp)
414
415 #endif /* __i386 */
416 #endif /* __lint */
417
418 /*
419 * Invalidate a single page table entry in the TLB
420 */
421
422 #if defined(__lint)
423
424 /* ARGSUSED */
425 void
426 mmu_tlbflush_entry(caddr_t m)
427 {}
428
429 #else /* __lint */
430
431 #if defined(__amd64)
432
433 ENTRY(mmu_tlbflush_entry)
434 invlpg (%rdi)
435 ret
436 SET_SIZE(mmu_tlbflush_entry)
437
438 #elif defined(__i386)
439
440 ENTRY(mmu_tlbflush_entry)
441 movl 4(%esp), %eax
442 invlpg (%eax)
443 ret
444 SET_SIZE(mmu_tlbflush_entry)
445
446 #endif /* __i386 */
447 #endif /* __lint */
448
449
450 /*
451 * Get/Set the value of various control registers
452 */
453
454 #if defined(__lint)
455
456 ulong_t
457 getcr0(void)
458 { return (0); }
459
460 /* ARGSUSED */
461 void
462 setcr0(ulong_t value)
463 {}
464
465 ulong_t
466 getcr2(void)
467 { return (0); }
468
469 ulong_t
470 getcr3(void)
471 { return (0); }
472
473 #if !defined(__xpv)
474 /* ARGSUSED */
475 void
476 setcr3(ulong_t val)
477 {}
478
479 void
480 reload_cr3(void)
481 {}
482 #endif
483
484 ulong_t
485 getcr4(void)
486 { return (0); }
487
488 /* ARGSUSED */
489 void
490 setcr4(ulong_t val)
491 {}
492
493 #if defined(__amd64)
494
495 ulong_t
496 getcr8(void)
497 { return (0); }
498
499 /* ARGSUSED */
500 void
501 setcr8(ulong_t val)
502 {}
503
504 #endif /* __amd64 */
505
506 #else /* __lint */
507
508 #if defined(__amd64)
509
510 ENTRY(getcr0)
511 movq %cr0, %rax
512 ret
513 SET_SIZE(getcr0)
514
515 ENTRY(setcr0)
516 movq %rdi, %cr0
517 ret
518 SET_SIZE(setcr0)
519
520 ENTRY(getcr2)
521 #if defined(__xpv)
522 movq %gs:CPU_VCPU_INFO, %rax
523 movq VCPU_INFO_ARCH_CR2(%rax), %rax
524 #else
525 movq %cr2, %rax
526 #endif
527 ret
528 SET_SIZE(getcr2)
529
530 ENTRY(getcr3)
531 movq %cr3, %rax
532 ret
533 SET_SIZE(getcr3)
534
535 #if !defined(__xpv)
536
537 ENTRY(setcr3)
538 movq %rdi, %cr3
539 ret
540 SET_SIZE(setcr3)
541
542 ENTRY(reload_cr3)
543 movq %cr3, %rdi
544 movq %rdi, %cr3
545 ret
546 SET_SIZE(reload_cr3)
547
548 #endif /* __xpv */
549
550 ENTRY(getcr4)
551 movq %cr4, %rax
552 ret
553 SET_SIZE(getcr4)
554
555 ENTRY(setcr4)
556 movq %rdi, %cr4
557 ret
558 SET_SIZE(setcr4)
559
560 ENTRY(getcr8)
561 movq %cr8, %rax
562 ret
563 SET_SIZE(getcr8)
564
565 ENTRY(setcr8)
566 movq %rdi, %cr8
567 ret
568 SET_SIZE(setcr8)
569
570 #elif defined(__i386)
571
572 ENTRY(getcr0)
573 movl %cr0, %eax
574 ret
575 SET_SIZE(getcr0)
576
577 ENTRY(setcr0)
578 movl 4(%esp), %eax
579 movl %eax, %cr0
580 ret
581 SET_SIZE(setcr0)
582
583 ENTRY(getcr2)
584 #if defined(__xpv)
585 movl %gs:CPU_VCPU_INFO, %eax
586 movl VCPU_INFO_ARCH_CR2(%eax), %eax
587 #else
588 movl %cr2, %eax
589 #endif
590 ret
591 SET_SIZE(getcr2)
592
593 ENTRY(getcr3)
594 movl %cr3, %eax
595 ret
596 SET_SIZE(getcr3)
597
598 #if !defined(__xpv)
599
600 ENTRY(setcr3)
601 movl 4(%esp), %eax
602 movl %eax, %cr3
603 ret
604 SET_SIZE(setcr3)
605
606 ENTRY(reload_cr3)
607 movl %cr3, %eax
608 movl %eax, %cr3
609 ret
610 SET_SIZE(reload_cr3)
611
612 #endif /* __xpv */
613
614 ENTRY(getcr4)
615 movl %cr4, %eax
616 ret
617 SET_SIZE(getcr4)
618
619 ENTRY(setcr4)
620 movl 4(%esp), %eax
621 movl %eax, %cr4
622 ret
623 SET_SIZE(setcr4)
624
625 #endif /* __i386 */
626 #endif /* __lint */
627
628 #if defined(__lint)
629
630 /*ARGSUSED*/
631 uint32_t
632 __cpuid_insn(struct cpuid_regs *regs)
633 { return (0); }
634
635 #else /* __lint */
636
637 #if defined(__amd64)
638
639 ENTRY(__cpuid_insn)
640 movq %rbx, %r8
641 movq %rcx, %r9
642 movq %rdx, %r11
643 movl (%rdi), %eax /* %eax = regs->cp_eax */
644 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
645 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
646 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
647 cpuid
648 movl %eax, (%rdi) /* regs->cp_eax = %eax */
649 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
650 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
651 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
652 movq %r8, %rbx
653 movq %r9, %rcx
654 movq %r11, %rdx
655 ret
656 SET_SIZE(__cpuid_insn)
657
658 #elif defined(__i386)
659
660 ENTRY(__cpuid_insn)
661 pushl %ebp
662 movl 0x8(%esp), %ebp /* %ebp = regs */
663 pushl %ebx
664 pushl %ecx
665 pushl %edx
666 movl (%ebp), %eax /* %eax = regs->cp_eax */
667 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
668 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
669 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
670 cpuid
671 movl %eax, (%ebp) /* regs->cp_eax = %eax */
672 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
673 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
674 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
675 popl %edx
676 popl %ecx
677 popl %ebx
678 popl %ebp
679 ret
680 SET_SIZE(__cpuid_insn)
681
682 #endif /* __i386 */
683 #endif /* __lint */
684
685 #if defined(__xpv)
686 /*
687 * Defined in C
688 */
689 #else
690
691 #if defined(__lint)
692
693 /*ARGSUSED*/
694 void
695 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
696 { return; }
697
698 #else /* __lint */
699
700 #if defined(__amd64)
701
702 ENTRY_NP(i86_monitor)
703 pushq %rbp
704 movq %rsp, %rbp
705 movq %rdi, %rax /* addr */
706 movq %rsi, %rcx /* extensions */
707 /* rdx contains input arg3: hints */
708 .byte 0x0f, 0x01, 0xc8 /* monitor */
709 leave
710 ret
711 SET_SIZE(i86_monitor)
712
713 #elif defined(__i386)
714
715 ENTRY_NP(i86_monitor)
716 pushl %ebp
717 movl %esp, %ebp
718 movl 0x8(%ebp),%eax /* addr */
719 movl 0xc(%ebp),%ecx /* extensions */
720 movl 0x10(%ebp),%edx /* hints */
721 .byte 0x0f, 0x01, 0xc8 /* monitor */
722 leave
723 ret
724 SET_SIZE(i86_monitor)
725
726 #endif /* __i386 */
727 #endif /* __lint */
728
729 #if defined(__lint)
730
731 /*ARGSUSED*/
732 void
733 i86_mwait(uint32_t data, uint32_t extensions)
734 { return; }
735
736 #else /* __lint */
737
738 #if defined(__amd64)
739
740 ENTRY_NP(i86_mwait)
741 pushq %rbp
742 movq %rsp, %rbp
743 movq %rdi, %rax /* data */
744 movq %rsi, %rcx /* extensions */
745 .byte 0x0f, 0x01, 0xc9 /* mwait */
746 leave
747 ret
748 SET_SIZE(i86_mwait)
749
750 #elif defined(__i386)
751
752 ENTRY_NP(i86_mwait)
753 pushl %ebp
754 movl %esp, %ebp
755 movl 0x8(%ebp),%eax /* data */
756 movl 0xc(%ebp),%ecx /* extensions */
757 .byte 0x0f, 0x01, 0xc9 /* mwait */
758 leave
759 ret
760 SET_SIZE(i86_mwait)
761
762 #endif /* __i386 */
763 #endif /* __lint */
764
765 #if defined(__lint)
766
767 hrtime_t
768 tsc_read(void)
769 {
770 return (0);
771 }
772
773 #else /* __lint */
774
775 #if defined(__amd64)
776
777 ENTRY_NP(tsc_read)
778 movq %rbx, %r11
779 movl $0, %eax
780 cpuid
781 rdtsc
782 movq %r11, %rbx
783 shlq $32, %rdx
784 orq %rdx, %rax
785 ret
786 .globl _tsc_mfence_start
787 _tsc_mfence_start:
788 mfence
789 rdtsc
790 shlq $32, %rdx
791 orq %rdx, %rax
792 ret
793 .globl _tsc_mfence_end
794 _tsc_mfence_end:
795 .globl _tscp_start
796 _tscp_start:
797 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
798 shlq $32, %rdx
799 orq %rdx, %rax
800 ret
801 .globl _tscp_end
802 _tscp_end:
803 .globl _no_rdtsc_start
804 _no_rdtsc_start:
805 xorl %edx, %edx
806 xorl %eax, %eax
807 ret
808 .globl _no_rdtsc_end
809 _no_rdtsc_end:
810 .globl _tsc_lfence_start
811 _tsc_lfence_start:
812 lfence
813 rdtsc
814 shlq $32, %rdx
815 orq %rdx, %rax
816 ret
817 .globl _tsc_lfence_end
818 _tsc_lfence_end:
819 SET_SIZE(tsc_read)
820
821 #else /* __i386 */
822
823 ENTRY_NP(tsc_read)
824 pushl %ebx
825 movl $0, %eax
826 cpuid
827 rdtsc
828 popl %ebx
829 ret
830 .globl _tsc_mfence_start
831 _tsc_mfence_start:
832 mfence
833 rdtsc
834 ret
835 .globl _tsc_mfence_end
836 _tsc_mfence_end:
837 .globl _tscp_start
838 _tscp_start:
839 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
840 ret
841 .globl _tscp_end
842 _tscp_end:
843 .globl _no_rdtsc_start
844 _no_rdtsc_start:
845 xorl %edx, %edx
846 xorl %eax, %eax
847 ret
848 .globl _no_rdtsc_end
849 _no_rdtsc_end:
850 .globl _tsc_lfence_start
851 _tsc_lfence_start:
852 lfence
853 rdtsc
854 ret
855 .globl _tsc_lfence_end
856 _tsc_lfence_end:
857 SET_SIZE(tsc_read)
858
859 #endif /* __i386 */
860
861 #endif /* __lint */
862
863
864 #endif /* __xpv */
865
866 #ifdef __lint
867 /*
868 * Do not use this function for obtaining clock tick. This
869 * is called by callers who do not need to have a guarenteed
870 * correct tick value. The proper routine to use is tsc_read().
871 */
872 hrtime_t
873 randtick(void)
874 {
875 return (0);
876 }
877 #else
878 #if defined(__amd64)
879 ENTRY_NP(randtick)
880 rdtsc
881 shlq $32, %rdx
882 orq %rdx, %rax
883 ret
884 SET_SIZE(randtick)
885 #else
886 ENTRY_NP(randtick)
887 rdtsc
888 ret
889 SET_SIZE(randtick)
890 #endif /* __i386 */
891 #endif /* __lint */
892 /*
893 * Insert entryp after predp in a doubly linked list.
894 */
895
896 #if defined(__lint)
897
898 /*ARGSUSED*/
899 void
900 _insque(caddr_t entryp, caddr_t predp)
901 {}
902
903 #else /* __lint */
904
905 #if defined(__amd64)
906
907 ENTRY(_insque)
908 movq (%rsi), %rax /* predp->forw */
909 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
910 movq %rax, (%rdi) /* entryp->forw = predp->forw */
911 movq %rdi, (%rsi) /* predp->forw = entryp */
912 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
913 ret
914 SET_SIZE(_insque)
915
916 #elif defined(__i386)
917
918 ENTRY(_insque)
919 movl 8(%esp), %edx
920 movl 4(%esp), %ecx
921 movl (%edx), %eax /* predp->forw */
922 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
923 movl %eax, (%ecx) /* entryp->forw = predp->forw */
924 movl %ecx, (%edx) /* predp->forw = entryp */
925 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
926 ret
927 SET_SIZE(_insque)
928
929 #endif /* __i386 */
930 #endif /* __lint */
931
932 /*
933 * Remove entryp from a doubly linked list
934 */
935
936 #if defined(__lint)
937
938 /*ARGSUSED*/
939 void
940 _remque(caddr_t entryp)
941 {}
942
943 #else /* __lint */
944
945 #if defined(__amd64)
946
947 ENTRY(_remque)
948 movq (%rdi), %rax /* entry->forw */
949 movq CPTRSIZE(%rdi), %rdx /* entry->back */
950 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
951 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
952 ret
953 SET_SIZE(_remque)
954
955 #elif defined(__i386)
956
957 ENTRY(_remque)
958 movl 4(%esp), %ecx
959 movl (%ecx), %eax /* entry->forw */
960 movl CPTRSIZE(%ecx), %edx /* entry->back */
961 movl %eax, (%edx) /* entry->back->forw = entry->forw */
962 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
963 ret
964 SET_SIZE(_remque)
965
966 #endif /* __i386 */
967 #endif /* __lint */
968
969 /*
970 * Returns the number of
971 * non-NULL bytes in string argument.
972 */
973
974 #if defined(__lint)
975
976 /* ARGSUSED */
977 size_t
978 strlen(const char *str)
979 { return (0); }
980
981 #else /* __lint */
982
983 #if defined(__amd64)
984
985 /*
986 * This is close to a simple transliteration of a C version of this
987 * routine. We should either just -make- this be a C version, or
988 * justify having it in assembler by making it significantly faster.
989 *
990 * size_t
991 * strlen(const char *s)
992 * {
993 * const char *s0;
994 * #if defined(DEBUG)
995 * if ((uintptr_t)s < KERNELBASE)
996 * panic(.str_panic_msg);
997 * #endif
998 * for (s0 = s; *s; s++)
999 * ;
1000 * return (s - s0);
1001 * }
1002 */
1003
1004 ENTRY(strlen)
1005 #ifdef DEBUG
1006 movq postbootkernelbase(%rip), %rax
1007 cmpq %rax, %rdi
1008 jae str_valid
1009 pushq %rbp
1010 movq %rsp, %rbp
1011 leaq .str_panic_msg(%rip), %rdi
1012 xorl %eax, %eax
1013 call panic
1014 #endif /* DEBUG */
1015 str_valid:
1016 cmpb $0, (%rdi)
1017 movq %rdi, %rax
1018 je .null_found
1019 .align 4
1020 .strlen_loop:
1021 incq %rdi
1022 cmpb $0, (%rdi)
1023 jne .strlen_loop
1024 .null_found:
1025 subq %rax, %rdi
1026 movq %rdi, %rax
1027 ret
1028 SET_SIZE(strlen)
1029
1030 #elif defined(__i386)
1031
1032 ENTRY(strlen)
1033 #ifdef DEBUG
1034 movl postbootkernelbase, %eax
1035 cmpl %eax, 4(%esp)
1036 jae str_valid
1037 pushl %ebp
1038 movl %esp, %ebp
1039 pushl $.str_panic_msg
1040 call panic
1041 #endif /* DEBUG */
1042
1043 str_valid:
1044 movl 4(%esp), %eax /* %eax = string address */
1045 testl $3, %eax /* if %eax not word aligned */
1046 jnz .not_word_aligned /* goto .not_word_aligned */
1047 .align 4
1048 .word_aligned:
1049 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1050 movl $0x7f7f7f7f, %ecx
1051 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1052 addl $4, %eax /* next word */
1053 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1054 orl %edx, %ecx /* %ecx |= %edx */
1055 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1056 cmpl $0x80808080, %ecx /* if no null byte in this word */
1057 je .word_aligned /* goto .word_aligned */
1058 subl $4, %eax /* post-incremented */
1059 .not_word_aligned:
1060 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1061 je .null_found /* goto .null_found */
1062 incl %eax /* next byte */
1063 testl $3, %eax /* if %eax not word aligned */
1064 jnz .not_word_aligned /* goto .not_word_aligned */
1065 jmp .word_aligned /* goto .word_aligned */
1066 .align 4
1067 .null_found:
1068 subl 4(%esp), %eax /* %eax -= string address */
1069 ret
1070 SET_SIZE(strlen)
1071
1072 #endif /* __i386 */
1073
1074 #ifdef DEBUG
1075 .text
1076 .str_panic_msg:
1077 .string "strlen: argument below kernelbase"
1078 #endif /* DEBUG */
1079
1080 #endif /* __lint */
1081
1082 /*
1083 * Berkeley 4.3 introduced symbolically named interrupt levels
1084 * as a way deal with priority in a machine independent fashion.
1085 * Numbered priorities are machine specific, and should be
1086 * discouraged where possible.
1087 *
1088 * Note, for the machine specific priorities there are
1089 * examples listed for devices that use a particular priority.
1090 * It should not be construed that all devices of that
1091 * type should be at that priority. It is currently were
1092 * the current devices fit into the priority scheme based
1093 * upon time criticalness.
1094 *
1095 * The underlying assumption of these assignments is that
1096 * IPL 10 is the highest level from which a device
1097 * routine can call wakeup. Devices that interrupt from higher
1098 * levels are restricted in what they can do. If they need
1099 * kernels services they should schedule a routine at a lower
1100 * level (via software interrupt) to do the required
1101 * processing.
1102 *
1103 * Examples of this higher usage:
1104 * Level Usage
1105 * 14 Profiling clock (and PROM uart polling clock)
1106 * 12 Serial ports
1107 *
1108 * The serial ports request lower level processing on level 6.
1109 *
1110 * Also, almost all splN routines (where N is a number or a
1111 * mnemonic) will do a RAISE(), on the assumption that they are
1112 * never used to lower our priority.
1113 * The exceptions are:
1114 * spl8() Because you can't be above 15 to begin with!
1115 * splzs() Because this is used at boot time to lower our
1116 * priority, to allow the PROM to poll the uart.
1117 * spl0() Used to lower priority to 0.
1118 */
1119
1120 #if defined(__lint)
1121
1122 int spl0(void) { return (0); }
1123 int spl6(void) { return (0); }
1124 int spl7(void) { return (0); }
1125 int spl8(void) { return (0); }
1126 int splhigh(void) { return (0); }
1127 int splhi(void) { return (0); }
1128 int splzs(void) { return (0); }
1129
1130 /* ARGSUSED */
1131 void
1132 splx(int level)
1133 {}
1134
1135 #else /* __lint */
1136
1137 #if defined(__amd64)
1138
1139 #define SETPRI(level) \
1140 movl $/**/level, %edi; /* new priority */ \
1141 jmp do_splx /* redirect to do_splx */
1142
1143 #define RAISE(level) \
1144 movl $/**/level, %edi; /* new priority */ \
1145 jmp splr /* redirect to splr */
1146
1147 #elif defined(__i386)
1148
1149 #define SETPRI(level) \
1150 pushl $/**/level; /* new priority */ \
1151 call do_splx; /* invoke common splx code */ \
1152 addl $4, %esp; /* unstack arg */ \
1153 ret
1154
1155 #define RAISE(level) \
1156 pushl $/**/level; /* new priority */ \
1157 call splr; /* invoke common splr code */ \
1158 addl $4, %esp; /* unstack args */ \
1159 ret
1160
1161 #endif /* __i386 */
1162
1163 /* locks out all interrupts, including memory errors */
1164 ENTRY(spl8)
1165 SETPRI(15)
1166 SET_SIZE(spl8)
1167
1168 /* just below the level that profiling runs */
1169 ENTRY(spl7)
1170 RAISE(13)
1171 SET_SIZE(spl7)
1172
1173 /* sun specific - highest priority onboard serial i/o asy ports */
1174 ENTRY(splzs)
1175 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1176 SET_SIZE(splzs)
1177
1178 ENTRY(splhi)
1179 ALTENTRY(splhigh)
1180 ALTENTRY(spl6)
1181 ALTENTRY(i_ddi_splhigh)
1182
1183 RAISE(DISP_LEVEL)
1184
1185 SET_SIZE(i_ddi_splhigh)
1186 SET_SIZE(spl6)
1187 SET_SIZE(splhigh)
1188 SET_SIZE(splhi)
1189
1190 /* allow all interrupts */
1191 ENTRY(spl0)
1192 SETPRI(0)
1193 SET_SIZE(spl0)
1194
1195
1196 /* splx implementation */
1197 ENTRY(splx)
1198 jmp do_splx /* redirect to common splx code */
1199 SET_SIZE(splx)
1200
1201 #endif /* __lint */
1202
1203 #if defined(__i386)
1204
1205 /*
1206 * Read and write the %gs register
1207 */
1208
1209 #if defined(__lint)
1210
1211 /*ARGSUSED*/
1212 uint16_t
1213 getgs(void)
1214 { return (0); }
1215
1216 /*ARGSUSED*/
1217 void
1218 setgs(uint16_t sel)
1219 {}
1220
1221 #else /* __lint */
1222
1223 ENTRY(getgs)
1224 clr %eax
1225 movw %gs, %ax
1226 ret
1227 SET_SIZE(getgs)
1228
1229 ENTRY(setgs)
1230 movw 4(%esp), %gs
1231 ret
1232 SET_SIZE(setgs)
1233
1234 #endif /* __lint */
1235 #endif /* __i386 */
1236
1237 #if defined(__lint)
1238
1239 void
1240 pc_reset(void)
1241 {}
1242
1243 void
1244 efi_reset(void)
1245 {}
1246
1247 #else /* __lint */
1248
1249 ENTRY(wait_500ms)
1250 push %ebx
1251 movl $50000, %ebx
1252 1:
1253 call tenmicrosec
1254 decl %ebx
1255 jnz 1b
1256 pop %ebx
1257 ret
1258 SET_SIZE(wait_500ms)
1259
1260 #define RESET_METHOD_KBC 1
1261 #define RESET_METHOD_PORT92 2
1262 #define RESET_METHOD_PCI 4
1263
1264 DGDEF3(pc_reset_methods, 4, 8)
1265 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1266
1267 ENTRY(pc_reset)
1268
1269 #if defined(__i386)
1270 testl $RESET_METHOD_KBC, pc_reset_methods
1271 #elif defined(__amd64)
1272 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1273 #endif
1274 jz 1f
1275
1276 /
1277 / Try the classic keyboard controller-triggered reset.
1278 /
1279 movw $0x64, %dx
1280 movb $0xfe, %al
1281 outb (%dx)
1282
1283 / Wait up to 500 milliseconds here for the keyboard controller
1284 / to pull the reset line. On some systems where the keyboard
1285 / controller is slow to pull the reset line, the next reset method
1286 / may be executed (which may be bad if those systems hang when the
1287 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1288 / and Ferrari 4000 (doesn't like the cf9 reset method))
1289
1290 call wait_500ms
1291
1292 1:
1293 #if defined(__i386)
1294 testl $RESET_METHOD_PORT92, pc_reset_methods
1295 #elif defined(__amd64)
1296 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1297 #endif
1298 jz 3f
1299
1300 /
1301 / Try port 0x92 fast reset
1302 /
1303 movw $0x92, %dx
1304 inb (%dx)
1305 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1306 je 1f
1307 testb $1, %al / If bit 0
1308 jz 2f / is clear, jump to perform the reset
1309 andb $0xfe, %al / otherwise,
1310 outb (%dx) / clear bit 0 first, then
1311 2:
1312 orb $1, %al / Set bit 0
1313 outb (%dx) / and reset the system
1314 1:
1315
1316 call wait_500ms
1317
1318 3:
1319 #if defined(__i386)
1320 testl $RESET_METHOD_PCI, pc_reset_methods
1321 #elif defined(__amd64)
1322 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1323 #endif
1324 jz 4f
1325
1326 / Try the PCI (soft) reset vector (should work on all modern systems,
1327 / but has been shown to cause problems on 450NX systems, and some newer
1328 / systems (e.g. ATI IXP400-equipped systems))
1329 / When resetting via this method, 2 writes are required. The first
1330 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1331 / power cycle).
1332 / The reset occurs on the second write, during bit 2's transition from
1333 / 0->1.
1334 movw $0xcf9, %dx
1335 movb $0x2, %al / Reset mode = hard, no power cycle
1336 outb (%dx)
1337 movb $0x6, %al
1338 outb (%dx)
1339
1340 call wait_500ms
1341
1342 4:
1343 /
1344 / port 0xcf9 failed also. Last-ditch effort is to
1345 / triple-fault the CPU.
1346 / Also, use triple fault for EFI firmware
1347 /
1348 ENTRY(efi_reset)
1349 #if defined(__amd64)
1350 pushq $0x0
1351 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1352 lidt (%rsp)
1353 #elif defined(__i386)
1354 pushl $0x0
1355 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1356 lidt (%esp)
1357 #endif
1358 int $0x0 / Trigger interrupt, generate triple-fault
1359
1360 cli
1361 hlt / Wait forever
1362 /*NOTREACHED*/
1363 SET_SIZE(efi_reset)
1364 SET_SIZE(pc_reset)
1365
1366 #endif /* __lint */
1367
1368 /*
1369 * C callable in and out routines
1370 */
1371
1372 #if defined(__lint)
1373
1374 /* ARGSUSED */
1375 void
1376 outl(int port_address, uint32_t val)
1377 {}
1378
1379 #else /* __lint */
1380
1381 #if defined(__amd64)
1382
1383 ENTRY(outl)
1384 movw %di, %dx
1385 movl %esi, %eax
1386 outl (%dx)
1387 ret
1388 SET_SIZE(outl)
1389
1390 #elif defined(__i386)
1391
1392 .set PORT, 4
1393 .set VAL, 8
1394
1395 ENTRY(outl)
1396 movw PORT(%esp), %dx
1397 movl VAL(%esp), %eax
1398 outl (%dx)
1399 ret
1400 SET_SIZE(outl)
1401
1402 #endif /* __i386 */
1403 #endif /* __lint */
1404
1405 #if defined(__lint)
1406
1407 /* ARGSUSED */
1408 void
1409 outw(int port_address, uint16_t val)
1410 {}
1411
1412 #else /* __lint */
1413
1414 #if defined(__amd64)
1415
1416 ENTRY(outw)
1417 movw %di, %dx
1418 movw %si, %ax
1419 D16 outl (%dx) /* XX64 why not outw? */
1420 ret
1421 SET_SIZE(outw)
1422
1423 #elif defined(__i386)
1424
1425 ENTRY(outw)
1426 movw PORT(%esp), %dx
1427 movw VAL(%esp), %ax
1428 D16 outl (%dx)
1429 ret
1430 SET_SIZE(outw)
1431
1432 #endif /* __i386 */
1433 #endif /* __lint */
1434
1435 #if defined(__lint)
1436
1437 /* ARGSUSED */
1438 void
1439 outb(int port_address, uint8_t val)
1440 {}
1441
1442 #else /* __lint */
1443
1444 #if defined(__amd64)
1445
1446 ENTRY(outb)
1447 movw %di, %dx
1448 movb %sil, %al
1449 outb (%dx)
1450 ret
1451 SET_SIZE(outb)
1452
1453 #elif defined(__i386)
1454
1455 ENTRY(outb)
1456 movw PORT(%esp), %dx
1457 movb VAL(%esp), %al
1458 outb (%dx)
1459 ret
1460 SET_SIZE(outb)
1461
1462 #endif /* __i386 */
1463 #endif /* __lint */
1464
1465 #if defined(__lint)
1466
1467 /* ARGSUSED */
1468 uint32_t
1469 inl(int port_address)
1470 { return (0); }
1471
1472 #else /* __lint */
1473
1474 #if defined(__amd64)
1475
1476 ENTRY(inl)
1477 xorl %eax, %eax
1478 movw %di, %dx
1479 inl (%dx)
1480 ret
1481 SET_SIZE(inl)
1482
1483 #elif defined(__i386)
1484
1485 ENTRY(inl)
1486 movw PORT(%esp), %dx
1487 inl (%dx)
1488 ret
1489 SET_SIZE(inl)
1490
1491 #endif /* __i386 */
1492 #endif /* __lint */
1493
1494 #if defined(__lint)
1495
1496 /* ARGSUSED */
1497 uint16_t
1498 inw(int port_address)
1499 { return (0); }
1500
1501 #else /* __lint */
1502
1503 #if defined(__amd64)
1504
1505 ENTRY(inw)
1506 xorl %eax, %eax
1507 movw %di, %dx
1508 D16 inl (%dx)
1509 ret
1510 SET_SIZE(inw)
1511
1512 #elif defined(__i386)
1513
1514 ENTRY(inw)
1515 subl %eax, %eax
1516 movw PORT(%esp), %dx
1517 D16 inl (%dx)
1518 ret
1519 SET_SIZE(inw)
1520
1521 #endif /* __i386 */
1522 #endif /* __lint */
1523
1524
1525 #if defined(__lint)
1526
1527 /* ARGSUSED */
1528 uint8_t
1529 inb(int port_address)
1530 { return (0); }
1531
1532 #else /* __lint */
1533
1534 #if defined(__amd64)
1535
1536 ENTRY(inb)
1537 xorl %eax, %eax
1538 movw %di, %dx
1539 inb (%dx)
1540 ret
1541 SET_SIZE(inb)
1542
1543 #elif defined(__i386)
1544
1545 ENTRY(inb)
1546 subl %eax, %eax
1547 movw PORT(%esp), %dx
1548 inb (%dx)
1549 ret
1550 SET_SIZE(inb)
1551
1552 #endif /* __i386 */
1553 #endif /* __lint */
1554
1555
1556 #if defined(__lint)
1557
1558 /* ARGSUSED */
1559 void
1560 repoutsw(int port, uint16_t *addr, int cnt)
1561 {}
1562
1563 #else /* __lint */
1564
1565 #if defined(__amd64)
1566
1567 ENTRY(repoutsw)
1568 movl %edx, %ecx
1569 movw %di, %dx
1570 rep
1571 D16 outsl
1572 ret
1573 SET_SIZE(repoutsw)
1574
1575 #elif defined(__i386)
1576
1577 /*
1578 * The arguments and saved registers are on the stack in the
1579 * following order:
1580 * | cnt | +16
1581 * | *addr | +12
1582 * | port | +8
1583 * | eip | +4
1584 * | esi | <-- %esp
1585 * If additional values are pushed onto the stack, make sure
1586 * to adjust the following constants accordingly.
1587 */
1588 .set PORT, 8
1589 .set ADDR, 12
1590 .set COUNT, 16
1591
1592 ENTRY(repoutsw)
1593 pushl %esi
1594 movl PORT(%esp), %edx
1595 movl ADDR(%esp), %esi
1596 movl COUNT(%esp), %ecx
1597 rep
1598 D16 outsl
1599 popl %esi
1600 ret
1601 SET_SIZE(repoutsw)
1602
1603 #endif /* __i386 */
1604 #endif /* __lint */
1605
1606
1607 #if defined(__lint)
1608
1609 /* ARGSUSED */
1610 void
1611 repinsw(int port_addr, uint16_t *addr, int cnt)
1612 {}
1613
1614 #else /* __lint */
1615
1616 #if defined(__amd64)
1617
1618 ENTRY(repinsw)
1619 movl %edx, %ecx
1620 movw %di, %dx
1621 rep
1622 D16 insl
1623 ret
1624 SET_SIZE(repinsw)
1625
1626 #elif defined(__i386)
1627
1628 ENTRY(repinsw)
1629 pushl %edi
1630 movl PORT(%esp), %edx
1631 movl ADDR(%esp), %edi
1632 movl COUNT(%esp), %ecx
1633 rep
1634 D16 insl
1635 popl %edi
1636 ret
1637 SET_SIZE(repinsw)
1638
1639 #endif /* __i386 */
1640 #endif /* __lint */
1641
1642
1643 #if defined(__lint)
1644
1645 /* ARGSUSED */
1646 void
1647 repinsb(int port, uint8_t *addr, int count)
1648 {}
1649
1650 #else /* __lint */
1651
1652 #if defined(__amd64)
1653
1654 ENTRY(repinsb)
1655 movl %edx, %ecx
1656 movw %di, %dx
1657 movq %rsi, %rdi
1658 rep
1659 insb
1660 ret
1661 SET_SIZE(repinsb)
1662
1663 #elif defined(__i386)
1664
1665 /*
1666 * The arguments and saved registers are on the stack in the
1667 * following order:
1668 * | cnt | +16
1669 * | *addr | +12
1670 * | port | +8
1671 * | eip | +4
1672 * | esi | <-- %esp
1673 * If additional values are pushed onto the stack, make sure
1674 * to adjust the following constants accordingly.
1675 */
1676 .set IO_PORT, 8
1677 .set IO_ADDR, 12
1678 .set IO_COUNT, 16
1679
1680 ENTRY(repinsb)
1681 pushl %edi
1682 movl IO_ADDR(%esp), %edi
1683 movl IO_COUNT(%esp), %ecx
1684 movl IO_PORT(%esp), %edx
1685 rep
1686 insb
1687 popl %edi
1688 ret
1689 SET_SIZE(repinsb)
1690
1691 #endif /* __i386 */
1692 #endif /* __lint */
1693
1694
1695 /*
1696 * Input a stream of 32-bit words.
1697 * NOTE: count is a DWORD count.
1698 */
1699 #if defined(__lint)
1700
1701 /* ARGSUSED */
1702 void
1703 repinsd(int port, uint32_t *addr, int count)
1704 {}
1705
1706 #else /* __lint */
1707
1708 #if defined(__amd64)
1709
1710 ENTRY(repinsd)
1711 movl %edx, %ecx
1712 movw %di, %dx
1713 movq %rsi, %rdi
1714 rep
1715 insl
1716 ret
1717 SET_SIZE(repinsd)
1718
1719 #elif defined(__i386)
1720
1721 ENTRY(repinsd)
1722 pushl %edi
1723 movl IO_ADDR(%esp), %edi
1724 movl IO_COUNT(%esp), %ecx
1725 movl IO_PORT(%esp), %edx
1726 rep
1727 insl
1728 popl %edi
1729 ret
1730 SET_SIZE(repinsd)
1731
1732 #endif /* __i386 */
1733 #endif /* __lint */
1734
1735 /*
1736 * Output a stream of bytes
1737 * NOTE: count is a byte count
1738 */
1739 #if defined(__lint)
1740
1741 /* ARGSUSED */
1742 void
1743 repoutsb(int port, uint8_t *addr, int count)
1744 {}
1745
1746 #else /* __lint */
1747
1748 #if defined(__amd64)
1749
1750 ENTRY(repoutsb)
1751 movl %edx, %ecx
1752 movw %di, %dx
1753 rep
1754 outsb
1755 ret
1756 SET_SIZE(repoutsb)
1757
1758 #elif defined(__i386)
1759
1760 ENTRY(repoutsb)
1761 pushl %esi
1762 movl IO_ADDR(%esp), %esi
1763 movl IO_COUNT(%esp), %ecx
1764 movl IO_PORT(%esp), %edx
1765 rep
1766 outsb
1767 popl %esi
1768 ret
1769 SET_SIZE(repoutsb)
1770
1771 #endif /* __i386 */
1772 #endif /* __lint */
1773
1774 /*
1775 * Output a stream of 32-bit words
1776 * NOTE: count is a DWORD count
1777 */
1778 #if defined(__lint)
1779
1780 /* ARGSUSED */
1781 void
1782 repoutsd(int port, uint32_t *addr, int count)
1783 {}
1784
1785 #else /* __lint */
1786
1787 #if defined(__amd64)
1788
1789 ENTRY(repoutsd)
1790 movl %edx, %ecx
1791 movw %di, %dx
1792 rep
1793 outsl
1794 ret
1795 SET_SIZE(repoutsd)
1796
1797 #elif defined(__i386)
1798
1799 ENTRY(repoutsd)
1800 pushl %esi
1801 movl IO_ADDR(%esp), %esi
1802 movl IO_COUNT(%esp), %ecx
1803 movl IO_PORT(%esp), %edx
1804 rep
1805 outsl
1806 popl %esi
1807 ret
1808 SET_SIZE(repoutsd)
1809
1810 #endif /* __i386 */
1811 #endif /* __lint */
1812
1813 /*
1814 * void int3(void)
1815 * void int18(void)
1816 * void int20(void)
1817 * void int_cmci(void)
1818 */
1819
1820 #if defined(__lint)
1821
1822 void
1823 int3(void)
1824 {}
1825
1826 void
1827 int18(void)
1828 {}
1829
1830 void
1831 int20(void)
1832 {}
1833
1834 void
1835 int_cmci(void)
1836 {}
1837
1838 #else /* __lint */
1839
1840 ENTRY(int3)
1841 int $T_BPTFLT
1842 ret
1843 SET_SIZE(int3)
1844
1845 ENTRY(int18)
1846 int $T_MCE
1847 ret
1848 SET_SIZE(int18)
1849
1850 ENTRY(int20)
1851 movl boothowto, %eax
1852 andl $RB_DEBUG, %eax
1853 jz 1f
1854
1855 int $T_DBGENTR
1856 1:
1857 rep; ret /* use 2 byte return instruction when branch target */
1858 /* AMD Software Optimization Guide - Section 6.2 */
1859 SET_SIZE(int20)
1860
1861 ENTRY(int_cmci)
1862 int $T_ENOEXTFLT
1863 ret
1864 SET_SIZE(int_cmci)
1865
1866 #endif /* __lint */
1867
1868 #if defined(__lint)
1869
1870 /* ARGSUSED */
1871 int
1872 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1873 { return (0); }
1874
1875 #else /* __lint */
1876
1877 #if defined(__amd64)
1878
1879 ENTRY(scanc)
1880 /* rdi == size */
1881 /* rsi == cp */
1882 /* rdx == table */
1883 /* rcx == mask */
1884 addq %rsi, %rdi /* end = &cp[size] */
1885 .scanloop:
1886 cmpq %rdi, %rsi /* while (cp < end */
1887 jnb .scandone
1888 movzbq (%rsi), %r8 /* %r8 = *cp */
1889 incq %rsi /* cp++ */
1890 testb %cl, (%r8, %rdx)
1891 jz .scanloop /* && (table[*cp] & mask) == 0) */
1892 decq %rsi /* (fix post-increment) */
1893 .scandone:
1894 movl %edi, %eax
1895 subl %esi, %eax /* return (end - cp) */
1896 ret
1897 SET_SIZE(scanc)
1898
1899 #elif defined(__i386)
1900
1901 ENTRY(scanc)
1902 pushl %edi
1903 pushl %esi
1904 movb 24(%esp), %cl /* mask = %cl */
1905 movl 16(%esp), %esi /* cp = %esi */
1906 movl 20(%esp), %edx /* table = %edx */
1907 movl %esi, %edi
1908 addl 12(%esp), %edi /* end = &cp[size]; */
1909 .scanloop:
1910 cmpl %edi, %esi /* while (cp < end */
1911 jnb .scandone
1912 movzbl (%esi), %eax /* %al = *cp */
1913 incl %esi /* cp++ */
1914 movb (%edx, %eax), %al /* %al = table[*cp] */
1915 testb %al, %cl
1916 jz .scanloop /* && (table[*cp] & mask) == 0) */
1917 dec %esi /* post-incremented */
1918 .scandone:
1919 movl %edi, %eax
1920 subl %esi, %eax /* return (end - cp) */
1921 popl %esi
1922 popl %edi
1923 ret
1924 SET_SIZE(scanc)
1925
1926 #endif /* __i386 */
1927 #endif /* __lint */
1928
1929 /*
1930 * Replacement functions for ones that are normally inlined.
1931 * In addition to the copy in i86.il, they are defined here just in case.
1932 */
1933
1934 #if defined(__lint)
1935
1936 ulong_t
1937 intr_clear(void)
1938 { return (0); }
1939
1940 ulong_t
1941 clear_int_flag(void)
1942 { return (0); }
1943
1944 #else /* __lint */
1945
1946 #if defined(__amd64)
1947
1948 ENTRY(intr_clear)
1949 ENTRY(clear_int_flag)
1950 pushfq
1951 popq %rax
1952 #if defined(__xpv)
1953 leaq xpv_panicking, %rdi
1954 movl (%rdi), %edi
1955 cmpl $0, %edi
1956 jne 2f
1957 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1958 /*
1959 * Synthesize the PS_IE bit from the event mask bit
1960 */
1961 andq $_BITNOT(PS_IE), %rax
1962 testb $1, %dl
1963 jnz 1f
1964 orq $PS_IE, %rax
1965 1:
1966 ret
1967 2:
1968 #endif
1969 CLI(%rdi)
1970 ret
1971 SET_SIZE(clear_int_flag)
1972 SET_SIZE(intr_clear)
1973
1974 #elif defined(__i386)
1975
1976 ENTRY(intr_clear)
1977 ENTRY(clear_int_flag)
1978 pushfl
1979 popl %eax
1980 #if defined(__xpv)
1981 leal xpv_panicking, %edx
1982 movl (%edx), %edx
1983 cmpl $0, %edx
1984 jne 2f
1985 CLIRET(%edx, %cl) /* returns event mask in %cl */
1986 /*
1987 * Synthesize the PS_IE bit from the event mask bit
1988 */
1989 andl $_BITNOT(PS_IE), %eax
1990 testb $1, %cl
1991 jnz 1f
1992 orl $PS_IE, %eax
1993 1:
1994 ret
1995 2:
1996 #endif
1997 CLI(%edx)
1998 ret
1999 SET_SIZE(clear_int_flag)
2000 SET_SIZE(intr_clear)
2001
2002 #endif /* __i386 */
2003 #endif /* __lint */
2004
2005 #if defined(__lint)
2006
2007 struct cpu *
2008 curcpup(void)
2009 { return 0; }
2010
2011 #else /* __lint */
2012
2013 #if defined(__amd64)
2014
2015 ENTRY(curcpup)
2016 movq %gs:CPU_SELF, %rax
2017 ret
2018 SET_SIZE(curcpup)
2019
2020 #elif defined(__i386)
2021
2022 ENTRY(curcpup)
2023 movl %gs:CPU_SELF, %eax
2024 ret
2025 SET_SIZE(curcpup)
2026
2027 #endif /* __i386 */
2028 #endif /* __lint */
2029
2030 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2031 * These functions reverse the byte order of the input parameter and returns
2032 * the result. This is to convert the byte order from host byte order
2033 * (little endian) to network byte order (big endian), or vice versa.
2034 */
2035
2036 #if defined(__lint)
2037
2038 uint64_t
2039 htonll(uint64_t i)
2040 { return (i); }
2041
2042 uint64_t
2043 ntohll(uint64_t i)
2044 { return (i); }
2045
2046 uint32_t
2047 htonl(uint32_t i)
2048 { return (i); }
2049
2050 uint32_t
2051 ntohl(uint32_t i)
2052 { return (i); }
2053
2054 uint16_t
2055 htons(uint16_t i)
2056 { return (i); }
2057
2058 uint16_t
2059 ntohs(uint16_t i)
2060 { return (i); }
2061
2062 #else /* __lint */
2063
2064 #if defined(__amd64)
2065
2066 ENTRY(htonll)
2067 ALTENTRY(ntohll)
2068 movq %rdi, %rax
2069 bswapq %rax
2070 ret
2071 SET_SIZE(ntohll)
2072 SET_SIZE(htonll)
2073
2074 /* XX64 there must be shorter sequences for this */
2075 ENTRY(htonl)
2076 ALTENTRY(ntohl)
2077 movl %edi, %eax
2078 bswap %eax
2079 ret
2080 SET_SIZE(ntohl)
2081 SET_SIZE(htonl)
2082
2083 /* XX64 there must be better sequences for this */
2084 ENTRY(htons)
2085 ALTENTRY(ntohs)
2086 movl %edi, %eax
2087 bswap %eax
2088 shrl $16, %eax
2089 ret
2090 SET_SIZE(ntohs)
2091 SET_SIZE(htons)
2092
2093 #elif defined(__i386)
2094
2095 ENTRY(htonll)
2096 ALTENTRY(ntohll)
2097 movl 4(%esp), %edx
2098 movl 8(%esp), %eax
2099 bswap %edx
2100 bswap %eax
2101 ret
2102 SET_SIZE(ntohll)
2103 SET_SIZE(htonll)
2104
2105 ENTRY(htonl)
2106 ALTENTRY(ntohl)
2107 movl 4(%esp), %eax
2108 bswap %eax
2109 ret
2110 SET_SIZE(ntohl)
2111 SET_SIZE(htonl)
2112
2113 ENTRY(htons)
2114 ALTENTRY(ntohs)
2115 movl 4(%esp), %eax
2116 bswap %eax
2117 shrl $16, %eax
2118 ret
2119 SET_SIZE(ntohs)
2120 SET_SIZE(htons)
2121
2122 #endif /* __i386 */
2123 #endif /* __lint */
2124
2125
2126 #if defined(__lint)
2127
2128 /* ARGSUSED */
2129 void
2130 intr_restore(ulong_t i)
2131 { return; }
2132
2133 /* ARGSUSED */
2134 void
2135 restore_int_flag(ulong_t i)
2136 { return; }
2137
2138 #else /* __lint */
2139
2140 #if defined(__amd64)
2141
2142 ENTRY(intr_restore)
2143 ENTRY(restore_int_flag)
2144 testq $PS_IE, %rdi
2145 jz 1f
2146 #if defined(__xpv)
2147 leaq xpv_panicking, %rsi
2148 movl (%rsi), %esi
2149 cmpl $0, %esi
2150 jne 1f
2151 /*
2152 * Since we're -really- running unprivileged, our attempt
2153 * to change the state of the IF bit will be ignored.
2154 * The virtual IF bit is tweaked by CLI and STI.
2155 */
2156 IE_TO_EVENT_MASK(%rsi, %rdi)
2157 #else
2158 sti
2159 #endif
2160 1:
2161 ret
2162 SET_SIZE(restore_int_flag)
2163 SET_SIZE(intr_restore)
2164
2165 #elif defined(__i386)
2166
2167 ENTRY(intr_restore)
2168 ENTRY(restore_int_flag)
2169 testl $PS_IE, 4(%esp)
2170 jz 1f
2171 #if defined(__xpv)
2172 leal xpv_panicking, %edx
2173 movl (%edx), %edx
2174 cmpl $0, %edx
2175 jne 1f
2176 /*
2177 * Since we're -really- running unprivileged, our attempt
2178 * to change the state of the IF bit will be ignored.
2179 * The virtual IF bit is tweaked by CLI and STI.
2180 */
2181 IE_TO_EVENT_MASK(%edx, 4(%esp))
2182 #else
2183 sti
2184 #endif
2185 1:
2186 ret
2187 SET_SIZE(restore_int_flag)
2188 SET_SIZE(intr_restore)
2189
2190 #endif /* __i386 */
2191 #endif /* __lint */
2192
2193 #if defined(__lint)
2194
2195 void
2196 sti(void)
2197 {}
2198
2199 void
2200 cli(void)
2201 {}
2202
2203 #else /* __lint */
2204
2205 ENTRY(sti)
2206 STI
2207 ret
2208 SET_SIZE(sti)
2209
2210 ENTRY(cli)
2211 #if defined(__amd64)
2212 CLI(%rax)
2213 #elif defined(__i386)
2214 CLI(%eax)
2215 #endif /* __i386 */
2216 ret
2217 SET_SIZE(cli)
2218
2219 #endif /* __lint */
2220
2221 #if defined(__lint)
2222
2223 dtrace_icookie_t
2224 dtrace_interrupt_disable(void)
2225 { return (0); }
2226
2227 #else /* __lint */
2228
2229 #if defined(__amd64)
2230
2231 ENTRY(dtrace_interrupt_disable)
2232 pushfq
2233 popq %rax
2234 #if defined(__xpv)
2235 leaq xpv_panicking, %rdi
2236 movl (%rdi), %edi
2237 cmpl $0, %edi
2238 jne 1f
2239 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2240 /*
2241 * Synthesize the PS_IE bit from the event mask bit
2242 */
2243 andq $_BITNOT(PS_IE), %rax
2244 testb $1, %dl
2245 jnz 1f
2246 orq $PS_IE, %rax
2247 1:
2248 #else
2249 CLI(%rdx)
2250 #endif
2251 ret
2252 SET_SIZE(dtrace_interrupt_disable)
2253
2254 #elif defined(__i386)
2255
2256 ENTRY(dtrace_interrupt_disable)
2257 pushfl
2258 popl %eax
2259 #if defined(__xpv)
2260 leal xpv_panicking, %edx
2261 movl (%edx), %edx
2262 cmpl $0, %edx
2263 jne 1f
2264 CLIRET(%edx, %cl) /* returns event mask in %cl */
2265 /*
2266 * Synthesize the PS_IE bit from the event mask bit
2267 */
2268 andl $_BITNOT(PS_IE), %eax
2269 testb $1, %cl
2270 jnz 1f
2271 orl $PS_IE, %eax
2272 1:
2273 #else
2274 CLI(%edx)
2275 #endif
2276 ret
2277 SET_SIZE(dtrace_interrupt_disable)
2278
2279 #endif /* __i386 */
2280 #endif /* __lint */
2281
2282 #if defined(__lint)
2283
2284 /*ARGSUSED*/
2285 void
2286 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2287 {}
2288
2289 #else /* __lint */
2290
2291 #if defined(__amd64)
2292
2293 ENTRY(dtrace_interrupt_enable)
2294 pushq %rdi
2295 popfq
2296 #if defined(__xpv)
2297 leaq xpv_panicking, %rdx
2298 movl (%rdx), %edx
2299 cmpl $0, %edx
2300 jne 1f
2301 /*
2302 * Since we're -really- running unprivileged, our attempt
2303 * to change the state of the IF bit will be ignored. The
2304 * virtual IF bit is tweaked by CLI and STI.
2305 */
2306 IE_TO_EVENT_MASK(%rdx, %rdi)
2307 #endif
2308 ret
2309 SET_SIZE(dtrace_interrupt_enable)
2310
2311 #elif defined(__i386)
2312
2313 ENTRY(dtrace_interrupt_enable)
2314 movl 4(%esp), %eax
2315 pushl %eax
2316 popfl
2317 #if defined(__xpv)
2318 leal xpv_panicking, %edx
2319 movl (%edx), %edx
2320 cmpl $0, %edx
2321 jne 1f
2322 /*
2323 * Since we're -really- running unprivileged, our attempt
2324 * to change the state of the IF bit will be ignored. The
2325 * virtual IF bit is tweaked by CLI and STI.
2326 */
2327 IE_TO_EVENT_MASK(%edx, %eax)
2328 #endif
2329 ret
2330 SET_SIZE(dtrace_interrupt_enable)
2331
2332 #endif /* __i386 */
2333 #endif /* __lint */
2334
2335
2336 #if defined(lint)
2337
2338 void
2339 dtrace_membar_producer(void)
2340 {}
2341
2342 void
2343 dtrace_membar_consumer(void)
2344 {}
2345
2346 #else /* __lint */
2347
2348 ENTRY(dtrace_membar_producer)
2349 rep; ret /* use 2 byte return instruction when branch target */
2350 /* AMD Software Optimization Guide - Section 6.2 */
2351 SET_SIZE(dtrace_membar_producer)
2352
2353 ENTRY(dtrace_membar_consumer)
2354 rep; ret /* use 2 byte return instruction when branch target */
2355 /* AMD Software Optimization Guide - Section 6.2 */
2356 SET_SIZE(dtrace_membar_consumer)
2357
2358 #endif /* __lint */
2359
2360 #if defined(__lint)
2361
2362 kthread_id_t
2363 threadp(void)
2364 { return ((kthread_id_t)0); }
2365
2366 #else /* __lint */
2367
2368 #if defined(__amd64)
2369
2370 ENTRY(threadp)
2371 movq %gs:CPU_THREAD, %rax
2372 ret
2373 SET_SIZE(threadp)
2374
2375 #elif defined(__i386)
2376
2377 ENTRY(threadp)
2378 movl %gs:CPU_THREAD, %eax
2379 ret
2380 SET_SIZE(threadp)
2381
2382 #endif /* __i386 */
2383 #endif /* __lint */
2384
2385 /*
2386 * Checksum routine for Internet Protocol Headers
2387 */
2388
2389 #if defined(__lint)
2390
2391 /* ARGSUSED */
2392 unsigned int
2393 ip_ocsum(
2394 ushort_t *address, /* ptr to 1st message buffer */
2395 int halfword_count, /* length of data */
2396 unsigned int sum) /* partial checksum */
2397 {
2398 int i;
2399 unsigned int psum = 0; /* partial sum */
2400
2401 for (i = 0; i < halfword_count; i++, address++) {
2402 psum += *address;
2403 }
2404
2405 while ((psum >> 16) != 0) {
2406 psum = (psum & 0xffff) + (psum >> 16);
2407 }
2408
2409 psum += sum;
2410
2411 while ((psum >> 16) != 0) {
2412 psum = (psum & 0xffff) + (psum >> 16);
2413 }
2414
2415 return (psum);
2416 }
2417
2418 #else /* __lint */
2419
2420 #if defined(__amd64)
2421
2422 ENTRY(ip_ocsum)
2423 pushq %rbp
2424 movq %rsp, %rbp
2425 #ifdef DEBUG
2426 movq postbootkernelbase(%rip), %rax
2427 cmpq %rax, %rdi
2428 jnb 1f
2429 xorl %eax, %eax
2430 movq %rdi, %rsi
2431 leaq .ip_ocsum_panic_msg(%rip), %rdi
2432 call panic
2433 /*NOTREACHED*/
2434 .ip_ocsum_panic_msg:
2435 .string "ip_ocsum: address 0x%p below kernelbase\n"
2436 1:
2437 #endif
2438 movl %esi, %ecx /* halfword_count */
2439 movq %rdi, %rsi /* address */
2440 /* partial sum in %edx */
2441 xorl %eax, %eax
2442 testl %ecx, %ecx
2443 jz .ip_ocsum_done
2444 testq $3, %rsi
2445 jnz .ip_csum_notaligned
2446 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2447 .next_iter:
2448 /* XX64 opportunities for prefetch? */
2449 /* XX64 compute csum with 64 bit quantities? */
2450 subl $32, %ecx
2451 jl .less_than_32
2452
2453 addl 0(%rsi), %edx
2454 .only60:
2455 adcl 4(%rsi), %eax
2456 .only56:
2457 adcl 8(%rsi), %edx
2458 .only52:
2459 adcl 12(%rsi), %eax
2460 .only48:
2461 adcl 16(%rsi), %edx
2462 .only44:
2463 adcl 20(%rsi), %eax
2464 .only40:
2465 adcl 24(%rsi), %edx
2466 .only36:
2467 adcl 28(%rsi), %eax
2468 .only32:
2469 adcl 32(%rsi), %edx
2470 .only28:
2471 adcl 36(%rsi), %eax
2472 .only24:
2473 adcl 40(%rsi), %edx
2474 .only20:
2475 adcl 44(%rsi), %eax
2476 .only16:
2477 adcl 48(%rsi), %edx
2478 .only12:
2479 adcl 52(%rsi), %eax
2480 .only8:
2481 adcl 56(%rsi), %edx
2482 .only4:
2483 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2484 .only0:
2485 adcl $0, %eax /* could be adding -1 in eax with a carry */
2486 adcl $0, %eax
2487
2488 addq $64, %rsi
2489 testl %ecx, %ecx
2490 jnz .next_iter
2491
2492 .ip_ocsum_done:
2493 addl %eax, %edx
2494 adcl $0, %edx
2495 movl %edx, %eax /* form a 16 bit checksum by */
2496 shrl $16, %eax /* adding two halves of 32 bit checksum */
2497 addw %dx, %ax
2498 adcw $0, %ax
2499 andl $0xffff, %eax
2500 leave
2501 ret
2502
2503 .ip_csum_notaligned:
2504 xorl %edi, %edi
2505 movw (%rsi), %di
2506 addl %edi, %edx
2507 adcl $0, %edx
2508 addq $2, %rsi
2509 decl %ecx
2510 jmp .ip_csum_aligned
2511
2512 .less_than_32:
2513 addl $32, %ecx
2514 testl $1, %ecx
2515 jz .size_aligned
2516 andl $0xfe, %ecx
2517 movzwl (%rsi, %rcx, 2), %edi
2518 addl %edi, %edx
2519 adcl $0, %edx
2520 .size_aligned:
2521 movl %ecx, %edi
2522 shrl $1, %ecx
2523 shl $1, %edi
2524 subq $64, %rdi
2525 addq %rdi, %rsi
2526 leaq .ip_ocsum_jmptbl(%rip), %rdi
2527 leaq (%rdi, %rcx, 8), %rdi
2528 xorl %ecx, %ecx
2529 clc
2530 jmp *(%rdi)
2531
2532 .align 8
2533 .ip_ocsum_jmptbl:
2534 .quad .only0, .only4, .only8, .only12, .only16, .only20
2535 .quad .only24, .only28, .only32, .only36, .only40, .only44
2536 .quad .only48, .only52, .only56, .only60
2537 SET_SIZE(ip_ocsum)
2538
2539 #elif defined(__i386)
2540
2541 ENTRY(ip_ocsum)
2542 pushl %ebp
2543 movl %esp, %ebp
2544 pushl %ebx
2545 pushl %esi
2546 pushl %edi
2547 movl 12(%ebp), %ecx /* count of half words */
2548 movl 16(%ebp), %edx /* partial checksum */
2549 movl 8(%ebp), %esi
2550 xorl %eax, %eax
2551 testl %ecx, %ecx
2552 jz .ip_ocsum_done
2553
2554 testl $3, %esi
2555 jnz .ip_csum_notaligned
2556 .ip_csum_aligned:
2557 .next_iter:
2558 subl $32, %ecx
2559 jl .less_than_32
2560
2561 addl 0(%esi), %edx
2562 .only60:
2563 adcl 4(%esi), %eax
2564 .only56:
2565 adcl 8(%esi), %edx
2566 .only52:
2567 adcl 12(%esi), %eax
2568 .only48:
2569 adcl 16(%esi), %edx
2570 .only44:
2571 adcl 20(%esi), %eax
2572 .only40:
2573 adcl 24(%esi), %edx
2574 .only36:
2575 adcl 28(%esi), %eax
2576 .only32:
2577 adcl 32(%esi), %edx
2578 .only28:
2579 adcl 36(%esi), %eax
2580 .only24:
2581 adcl 40(%esi), %edx
2582 .only20:
2583 adcl 44(%esi), %eax
2584 .only16:
2585 adcl 48(%esi), %edx
2586 .only12:
2587 adcl 52(%esi), %eax
2588 .only8:
2589 adcl 56(%esi), %edx
2590 .only4:
2591 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2592 .only0:
2593 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2594 adcl $0, %eax
2595
2596 addl $64, %esi
2597 andl %ecx, %ecx
2598 jnz .next_iter
2599
2600 .ip_ocsum_done:
2601 addl %eax, %edx
2602 adcl $0, %edx
2603 movl %edx, %eax /* form a 16 bit checksum by */
2604 shrl $16, %eax /* adding two halves of 32 bit checksum */
2605 addw %dx, %ax
2606 adcw $0, %ax
2607 andl $0xffff, %eax
2608 popl %edi /* restore registers */
2609 popl %esi
2610 popl %ebx
2611 leave
2612 ret
2613
2614 .ip_csum_notaligned:
2615 xorl %edi, %edi
2616 movw (%esi), %di
2617 addl %edi, %edx
2618 adcl $0, %edx
2619 addl $2, %esi
2620 decl %ecx
2621 jmp .ip_csum_aligned
2622
2623 .less_than_32:
2624 addl $32, %ecx
2625 testl $1, %ecx
2626 jz .size_aligned
2627 andl $0xfe, %ecx
2628 movzwl (%esi, %ecx, 2), %edi
2629 addl %edi, %edx
2630 adcl $0, %edx
2631 .size_aligned:
2632 movl %ecx, %edi
2633 shrl $1, %ecx
2634 shl $1, %edi
2635 subl $64, %edi
2636 addl %edi, %esi
2637 movl $.ip_ocsum_jmptbl, %edi
2638 lea (%edi, %ecx, 4), %edi
2639 xorl %ecx, %ecx
2640 clc
2641 jmp *(%edi)
2642 SET_SIZE(ip_ocsum)
2643
2644 .data
2645 .align 4
2646
2647 .ip_ocsum_jmptbl:
2648 .long .only0, .only4, .only8, .only12, .only16, .only20
2649 .long .only24, .only28, .only32, .only36, .only40, .only44
2650 .long .only48, .only52, .only56, .only60
2651
2652
2653 #endif /* __i386 */
2654 #endif /* __lint */
2655
2656 /*
2657 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2658 * Provided to manipulate hrtime_t values.
2659 */
2660 #if defined(__lint)
2661
2662 /* result = a * b; */
2663
2664 /* ARGSUSED */
2665 unsigned long long
2666 mul32(uint_t a, uint_t b)
2667 { return (0); }
2668
2669 #else /* __lint */
2670
2671 #if defined(__amd64)
2672
2673 ENTRY(mul32)
2674 xorl %edx, %edx /* XX64 joe, paranoia? */
2675 movl %edi, %eax
2676 mull %esi
2677 shlq $32, %rdx
2678 orq %rdx, %rax
2679 ret
2680 SET_SIZE(mul32)
2681
2682 #elif defined(__i386)
2683
2684 ENTRY(mul32)
2685 movl 8(%esp), %eax
2686 movl 4(%esp), %ecx
2687 mull %ecx
2688 ret
2689 SET_SIZE(mul32)
2690
2691 #endif /* __i386 */
2692 #endif /* __lint */
2693
2694 #if defined(notused)
2695 #if defined(__lint)
2696 /* ARGSUSED */
2697 void
2698 load_pte64(uint64_t *pte, uint64_t pte_value)
2699 {}
2700 #else /* __lint */
2701 .globl load_pte64
2702 load_pte64:
2703 movl 4(%esp), %eax
2704 movl 8(%esp), %ecx
2705 movl 12(%esp), %edx
2706 movl %edx, 4(%eax)
2707 movl %ecx, (%eax)
2708 ret
2709 #endif /* __lint */
2710 #endif /* notused */
2711
2712 #if defined(__lint)
2713
2714 /*ARGSUSED*/
2715 void
2716 scan_memory(caddr_t addr, size_t size)
2717 {}
2718
2719 #else /* __lint */
2720
2721 #if defined(__amd64)
2722
2723 ENTRY(scan_memory)
2724 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2725 jz .scanm_done
2726 movq %rsi, %rcx /* move count into rep control register */
2727 movq %rdi, %rsi /* move addr into lodsq control reg. */
2728 rep lodsq /* scan the memory range */
2729 .scanm_done:
2730 rep; ret /* use 2 byte return instruction when branch target */
2731 /* AMD Software Optimization Guide - Section 6.2 */
2732 SET_SIZE(scan_memory)
2733
2734 #elif defined(__i386)
2735
2736 ENTRY(scan_memory)
2737 pushl %ecx
2738 pushl %esi
2739 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2740 shrl $2, %ecx /* convert from byte count to word count */
2741 jz .scanm_done
2742 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2743 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2744 lodsl
2745 .scanm_done:
2746 popl %esi
2747 popl %ecx
2748 ret
2749 SET_SIZE(scan_memory)
2750
2751 #endif /* __i386 */
2752 #endif /* __lint */
2753
2754
2755 #if defined(__lint)
2756
2757 /*ARGSUSED */
2758 int
2759 lowbit(ulong_t i)
2760 { return (0); }
2761
2762 #else /* __lint */
2763
2764 #if defined(__amd64)
2765
2766 ENTRY(lowbit)
2767 movl $-1, %eax
2768 bsfq %rdi, %rax
2769 incl %eax
2770 ret
2771 SET_SIZE(lowbit)
2772
2773 #elif defined(__i386)
2774
2775 ENTRY(lowbit)
2776 movl $-1, %eax
2777 bsfl 4(%esp), %eax
2778 incl %eax
2779 ret
2780 SET_SIZE(lowbit)
2781
2782 #endif /* __i386 */
2783 #endif /* __lint */
2784
2785 #if defined(__lint)
2786
2787 /*ARGSUSED*/
2788 int
2789 highbit(ulong_t i)
2790 { return (0); }
2791
2792 #else /* __lint */
2793
2794 #if defined(__amd64)
2795
2796 ENTRY(highbit)
2797 movl $-1, %eax
2798 bsrq %rdi, %rax
2799 incl %eax
2800 ret
2801 SET_SIZE(highbit)
2802
2803 #elif defined(__i386)
2804
2805 ENTRY(highbit)
2806 movl $-1, %eax
2807 bsrl 4(%esp), %eax
2808 incl %eax
2809 ret
2810 SET_SIZE(highbit)
2811
2812 #endif /* __i386 */
2813 #endif /* __lint */
2814
2815 #if defined(__lint)
2816
2817 /*ARGSUSED*/
2818 uint64_t
2819 rdmsr(uint_t r)
2820 { return (0); }
2821
2822 /*ARGSUSED*/
2823 void
2824 wrmsr(uint_t r, const uint64_t val)
2825 {}
2826
2827 /*ARGSUSED*/
2828 uint64_t
2829 xrdmsr(uint_t r)
2830 { return (0); }
2831
2832 /*ARGSUSED*/
2833 void
2834 xwrmsr(uint_t r, const uint64_t val)
2835 {}
2836
2837 void
2838 invalidate_cache(void)
2839 {}
2840
2841 #else /* __lint */
2842
2843 #define XMSR_ACCESS_VAL $0x9c5a203a
2844
2845 #if defined(__amd64)
2846
2847 ENTRY(rdmsr)
2848 movl %edi, %ecx
2849 rdmsr
2850 shlq $32, %rdx
2851 orq %rdx, %rax
2852 ret
2853 SET_SIZE(rdmsr)
2854
2855 ENTRY(wrmsr)
2856 movq %rsi, %rdx
2857 shrq $32, %rdx
2858 movl %esi, %eax
2859 movl %edi, %ecx
2860 wrmsr
2861 ret
2862 SET_SIZE(wrmsr)
2863
2864 ENTRY(xrdmsr)
2865 pushq %rbp
2866 movq %rsp, %rbp
2867 movl %edi, %ecx
2868 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2869 rdmsr
2870 shlq $32, %rdx
2871 orq %rdx, %rax
2872 leave
2873 ret
2874 SET_SIZE(xrdmsr)
2875
2876 ENTRY(xwrmsr)
2877 pushq %rbp
2878 movq %rsp, %rbp
2879 movl %edi, %ecx
2880 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2881 movq %rsi, %rdx
2882 shrq $32, %rdx
2883 movl %esi, %eax
2884 wrmsr
2885 leave
2886 ret
2887 SET_SIZE(xwrmsr)
2888
2889 #elif defined(__i386)
2890
2891 ENTRY(rdmsr)
2892 movl 4(%esp), %ecx
2893 rdmsr
2894 ret
2895 SET_SIZE(rdmsr)
2896
2897 ENTRY(wrmsr)
2898 movl 4(%esp), %ecx
2899 movl 8(%esp), %eax
2900 movl 12(%esp), %edx
2901 wrmsr
2902 ret
2903 SET_SIZE(wrmsr)
2904
2905 ENTRY(xrdmsr)
2906 pushl %ebp
2907 movl %esp, %ebp
2908 movl 8(%esp), %ecx
2909 pushl %edi
2910 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2911 rdmsr
2912 popl %edi
2913 leave
2914 ret
2915 SET_SIZE(xrdmsr)
2916
2917 ENTRY(xwrmsr)
2918 pushl %ebp
2919 movl %esp, %ebp
2920 movl 8(%esp), %ecx
2921 movl 12(%esp), %eax
2922 movl 16(%esp), %edx
2923 pushl %edi
2924 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2925 wrmsr
2926 popl %edi
2927 leave
2928 ret
2929 SET_SIZE(xwrmsr)
2930
2931 #endif /* __i386 */
2932
2933 ENTRY(invalidate_cache)
2934 wbinvd
2935 ret
2936 SET_SIZE(invalidate_cache)
2937
2938 #endif /* __lint */
2939
2940 #if defined(__lint)
2941
2942 /*ARGSUSED*/
2943 void
2944 getcregs(struct cregs *crp)
2945 {}
2946
2947 #else /* __lint */
2948
2949 #if defined(__amd64)
2950
2951 ENTRY_NP(getcregs)
2952 #if defined(__xpv)
2953 /*
2954 * Only a few of the hardware control registers or descriptor tables
2955 * are directly accessible to us, so just zero the structure.
2956 *
2957 * XXPV Perhaps it would be helpful for the hypervisor to return
2958 * virtualized versions of these for post-mortem use.
2959 * (Need to reevaluate - perhaps it already does!)
2960 */
2961 pushq %rdi /* save *crp */
2962 movq $CREGSZ, %rsi
2963 call bzero
2964 popq %rdi
2965
2966 /*
2967 * Dump what limited information we can
2968 */
2969 movq %cr0, %rax
2970 movq %rax, CREG_CR0(%rdi) /* cr0 */
2971 movq %cr2, %rax
2972 movq %rax, CREG_CR2(%rdi) /* cr2 */
2973 movq %cr3, %rax
2974 movq %rax, CREG_CR3(%rdi) /* cr3 */
2975 movq %cr4, %rax
2976 movq %rax, CREG_CR4(%rdi) /* cr4 */
2977
2978 #else /* __xpv */
2979
2980 #define GETMSR(r, off, d) \
2981 movl $r, %ecx; \
2982 rdmsr; \
2983 movl %eax, off(d); \
2984 movl %edx, off+4(d)
2985
2986 xorl %eax, %eax
2987 movq %rax, CREG_GDT+8(%rdi)
2988 sgdt CREG_GDT(%rdi) /* 10 bytes */
2989 movq %rax, CREG_IDT+8(%rdi)
2990 sidt CREG_IDT(%rdi) /* 10 bytes */
2991 movq %rax, CREG_LDT(%rdi)
2992 sldt CREG_LDT(%rdi) /* 2 bytes */
2993 movq %rax, CREG_TASKR(%rdi)
2994 str CREG_TASKR(%rdi) /* 2 bytes */
2995 movq %cr0, %rax
2996 movq %rax, CREG_CR0(%rdi) /* cr0 */
2997 movq %cr2, %rax
2998 movq %rax, CREG_CR2(%rdi) /* cr2 */
2999 movq %cr3, %rax
3000 movq %rax, CREG_CR3(%rdi) /* cr3 */
3001 movq %cr4, %rax
3002 movq %rax, CREG_CR4(%rdi) /* cr4 */
3003 movq %cr8, %rax
3004 movq %rax, CREG_CR8(%rdi) /* cr8 */
3005 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3006 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3007 #endif /* __xpv */
3008 ret
3009 SET_SIZE(getcregs)
3010
3011 #undef GETMSR
3012
3013 #elif defined(__i386)
3014
3015 ENTRY_NP(getcregs)
3016 #if defined(__xpv)
3017 /*
3018 * Only a few of the hardware control registers or descriptor tables
3019 * are directly accessible to us, so just zero the structure.
3020 *
3021 * XXPV Perhaps it would be helpful for the hypervisor to return
3022 * virtualized versions of these for post-mortem use.
3023 * (Need to reevaluate - perhaps it already does!)
3024 */
3025 movl 4(%esp), %edx
3026 pushl $CREGSZ
3027 pushl %edx
3028 call bzero
3029 addl $8, %esp
3030 movl 4(%esp), %edx
3031
3032 /*
3033 * Dump what limited information we can
3034 */
3035 movl %cr0, %eax
3036 movl %eax, CREG_CR0(%edx) /* cr0 */
3037 movl %cr2, %eax
3038 movl %eax, CREG_CR2(%edx) /* cr2 */
3039 movl %cr3, %eax
3040 movl %eax, CREG_CR3(%edx) /* cr3 */
3041 movl %cr4, %eax
3042 movl %eax, CREG_CR4(%edx) /* cr4 */
3043
3044 #else /* __xpv */
3045
3046 movl 4(%esp), %edx
3047 movw $0, CREG_GDT+6(%edx)
3048 movw $0, CREG_IDT+6(%edx)
3049 sgdt CREG_GDT(%edx) /* gdt */
3050 sidt CREG_IDT(%edx) /* idt */
3051 sldt CREG_LDT(%edx) /* ldt */
3052 str CREG_TASKR(%edx) /* task */
3053 movl %cr0, %eax
3054 movl %eax, CREG_CR0(%edx) /* cr0 */
3055 movl %cr2, %eax
3056 movl %eax, CREG_CR2(%edx) /* cr2 */
3057 movl %cr3, %eax
3058 movl %eax, CREG_CR3(%edx) /* cr3 */
3059 testl $X86_LARGEPAGE, x86_feature
3060 jz .nocr4
3061 movl %cr4, %eax
3062 movl %eax, CREG_CR4(%edx) /* cr4 */
3063 jmp .skip
3064 .nocr4:
3065 movl $0, CREG_CR4(%edx)
3066 .skip:
3067 #endif
3068 ret
3069 SET_SIZE(getcregs)
3070
3071 #endif /* __i386 */
3072 #endif /* __lint */
3073
3074
3075 /*
3076 * A panic trigger is a word which is updated atomically and can only be set
3077 * once. We atomically store 0xDEFACEDD and load the old value. If the
3078 * previous value was 0, we succeed and return 1; otherwise return 0.
3079 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3080 * has its own version of this function to allow it to panic correctly from
3081 * probe context.
3082 */
3083 #if defined(__lint)
3084
3085 /*ARGSUSED*/
3086 int
3087 panic_trigger(int *tp)
3088 { return (0); }
3089
3090 /*ARGSUSED*/
3091 int
3092 dtrace_panic_trigger(int *tp)
3093 { return (0); }
3094
3095 #else /* __lint */
3096
3097 #if defined(__amd64)
3098
3099 ENTRY_NP(panic_trigger)
3100 xorl %eax, %eax
3101 movl $0xdefacedd, %edx
3102 lock
3103 xchgl %edx, (%rdi)
3104 cmpl $0, %edx
3105 je 0f
3106 movl $0, %eax
3107 ret
3108 0: movl $1, %eax
3109 ret
3110 SET_SIZE(panic_trigger)
3111
3112 ENTRY_NP(dtrace_panic_trigger)
3113 xorl %eax, %eax
3114 movl $0xdefacedd, %edx
3115 lock
3116 xchgl %edx, (%rdi)
3117 cmpl $0, %edx
3118 je 0f
3119 movl $0, %eax
3120 ret
3121 0: movl $1, %eax
3122 ret
3123 SET_SIZE(dtrace_panic_trigger)
3124
3125 #elif defined(__i386)
3126
3127 ENTRY_NP(panic_trigger)
3128 movl 4(%esp), %edx / %edx = address of trigger
3129 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3130 lock / assert lock
3131 xchgl %eax, (%edx) / exchange %eax and the trigger
3132 cmpl $0, %eax / if (%eax == 0x0)
3133 je 0f / return (1);
3134 movl $0, %eax / else
3135 ret / return (0);
3136 0: movl $1, %eax
3137 ret
3138 SET_SIZE(panic_trigger)
3139
3140 ENTRY_NP(dtrace_panic_trigger)
3141 movl 4(%esp), %edx / %edx = address of trigger
3142 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3143 lock / assert lock
3144 xchgl %eax, (%edx) / exchange %eax and the trigger
3145 cmpl $0, %eax / if (%eax == 0x0)
3146 je 0f / return (1);
3147 movl $0, %eax / else
3148 ret / return (0);
3149 0: movl $1, %eax
3150 ret
3151 SET_SIZE(dtrace_panic_trigger)
3152
3153 #endif /* __i386 */
3154 #endif /* __lint */
3155
3156 /*
3157 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3158 * into the panic code implemented in panicsys(). vpanic() is responsible
3159 * for passing through the format string and arguments, and constructing a
3160 * regs structure on the stack into which it saves the current register
3161 * values. If we are not dying due to a fatal trap, these registers will
3162 * then be preserved in panicbuf as the current processor state. Before
3163 * invoking panicsys(), vpanic() activates the first panic trigger (see
3164 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3165 * DTrace takes a slightly different panic path if it must panic from probe
3166 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3167 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3168 * branches back into vpanic().
3169 */
3170 #if defined(__lint)
3171
3172 /*ARGSUSED*/
3173 void
3174 vpanic(const char *format, va_list alist)
3175 {}
3176
3177 /*ARGSUSED*/
3178 void
3179 dtrace_vpanic(const char *format, va_list alist)
3180 {}
3181
3182 #else /* __lint */
3183
3184 #if defined(__amd64)
3185
3186 ENTRY_NP(vpanic) /* Initial stack layout: */
3187
3188 pushq %rbp /* | %rip | 0x60 */
3189 movq %rsp, %rbp /* | %rbp | 0x58 */
3190 pushfq /* | rfl | 0x50 */
3191 pushq %r11 /* | %r11 | 0x48 */
3192 pushq %r10 /* | %r10 | 0x40 */
3193 pushq %rbx /* | %rbx | 0x38 */
3194 pushq %rax /* | %rax | 0x30 */
3195 pushq %r9 /* | %r9 | 0x28 */
3196 pushq %r8 /* | %r8 | 0x20 */
3197 pushq %rcx /* | %rcx | 0x18 */
3198 pushq %rdx /* | %rdx | 0x10 */
3199 pushq %rsi /* | %rsi | 0x8 alist */
3200 pushq %rdi /* | %rdi | 0x0 format */
3201
3202 movq %rsp, %rbx /* %rbx = current %rsp */
3203
3204 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3205 call panic_trigger /* %eax = panic_trigger() */
3206
3207 vpanic_common:
3208 /*
3209 * The panic_trigger result is in %eax from the call above, and
3210 * dtrace_panic places it in %eax before branching here.
3211 * The rdmsr instructions that follow below will clobber %eax so
3212 * we stash the panic_trigger result in %r11d.
3213 */
3214 movl %eax, %r11d
3215 cmpl $0, %r11d
3216 je 0f
3217
3218 /*
3219 * If panic_trigger() was successful, we are the first to initiate a
3220 * panic: we now switch to the reserved panic_stack before continuing.
3221 */
3222 leaq panic_stack(%rip), %rsp
3223 addq $PANICSTKSIZE, %rsp
3224 0: subq $REGSIZE, %rsp
3225 /*
3226 * Now that we've got everything set up, store the register values as
3227 * they were when we entered vpanic() to the designated location in
3228 * the regs structure we allocated on the stack.
3229 */
3230 movq 0x0(%rbx), %rcx
3231 movq %rcx, REGOFF_RDI(%rsp)
3232 movq 0x8(%rbx), %rcx
3233 movq %rcx, REGOFF_RSI(%rsp)
3234 movq 0x10(%rbx), %rcx
3235 movq %rcx, REGOFF_RDX(%rsp)
3236 movq 0x18(%rbx), %rcx
3237 movq %rcx, REGOFF_RCX(%rsp)
3238 movq 0x20(%rbx), %rcx
3239
3240 movq %rcx, REGOFF_R8(%rsp)
3241 movq 0x28(%rbx), %rcx
3242 movq %rcx, REGOFF_R9(%rsp)
3243 movq 0x30(%rbx), %rcx
3244 movq %rcx, REGOFF_RAX(%rsp)
3245 movq 0x38(%rbx), %rcx
3246 movq %rcx, REGOFF_RBX(%rsp)
3247 movq 0x58(%rbx), %rcx
3248
3249 movq %rcx, REGOFF_RBP(%rsp)
3250 movq 0x40(%rbx), %rcx
3251 movq %rcx, REGOFF_R10(%rsp)
3252 movq 0x48(%rbx), %rcx
3253 movq %rcx, REGOFF_R11(%rsp)
3254 movq %r12, REGOFF_R12(%rsp)
3255
3256 movq %r13, REGOFF_R13(%rsp)
3257 movq %r14, REGOFF_R14(%rsp)
3258 movq %r15, REGOFF_R15(%rsp)
3259
3260 xorl %ecx, %ecx
3261 movw %ds, %cx
3262 movq %rcx, REGOFF_DS(%rsp)
3263 movw %es, %cx
3264 movq %rcx, REGOFF_ES(%rsp)
3265 movw %fs, %cx
3266 movq %rcx, REGOFF_FS(%rsp)
3267 movw %gs, %cx
3268 movq %rcx, REGOFF_GS(%rsp)
3269
3270 movq $0, REGOFF_TRAPNO(%rsp)
3271
3272 movq $0, REGOFF_ERR(%rsp)
3273 leaq vpanic(%rip), %rcx
3274 movq %rcx, REGOFF_RIP(%rsp)
3275 movw %cs, %cx
3276 movzwq %cx, %rcx
3277 movq %rcx, REGOFF_CS(%rsp)
3278 movq 0x50(%rbx), %rcx
3279 movq %rcx, REGOFF_RFL(%rsp)
3280 movq %rbx, %rcx
3281 addq $0x60, %rcx
3282 movq %rcx, REGOFF_RSP(%rsp)
3283 movw %ss, %cx
3284 movzwq %cx, %rcx
3285 movq %rcx, REGOFF_SS(%rsp)
3286
3287 /*
3288 * panicsys(format, alist, rp, on_panic_stack)
3289 */
3290 movq REGOFF_RDI(%rsp), %rdi /* format */
3291 movq REGOFF_RSI(%rsp), %rsi /* alist */
3292 movq %rsp, %rdx /* struct regs */
3293 movl %r11d, %ecx /* on_panic_stack */
3294 call panicsys
3295 addq $REGSIZE, %rsp
3296 popq %rdi
3297 popq %rsi
3298 popq %rdx
3299 popq %rcx
3300 popq %r8
3301 popq %r9
3302 popq %rax
3303 popq %rbx
3304 popq %r10
3305 popq %r11
3306 popfq
3307 leave
3308 ret
3309 SET_SIZE(vpanic)
3310
3311 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3312
3313 pushq %rbp /* | %rip | 0x60 */
3314 movq %rsp, %rbp /* | %rbp | 0x58 */
3315 pushfq /* | rfl | 0x50 */
3316 pushq %r11 /* | %r11 | 0x48 */
3317 pushq %r10 /* | %r10 | 0x40 */
3318 pushq %rbx /* | %rbx | 0x38 */
3319 pushq %rax /* | %rax | 0x30 */
3320 pushq %r9 /* | %r9 | 0x28 */
3321 pushq %r8 /* | %r8 | 0x20 */
3322 pushq %rcx /* | %rcx | 0x18 */
3323 pushq %rdx /* | %rdx | 0x10 */
3324 pushq %rsi /* | %rsi | 0x8 alist */
3325 pushq %rdi /* | %rdi | 0x0 format */
3326
3327 movq %rsp, %rbx /* %rbx = current %rsp */
3328
3329 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3330 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3331 jmp vpanic_common
3332
3333 SET_SIZE(dtrace_vpanic)
3334
3335 #elif defined(__i386)
3336
3337 ENTRY_NP(vpanic) / Initial stack layout:
3338
3339 pushl %ebp / | %eip | 20
3340 movl %esp, %ebp / | %ebp | 16
3341 pushl %eax / | %eax | 12
3342 pushl %ebx / | %ebx | 8
3343 pushl %ecx / | %ecx | 4
3344 pushl %edx / | %edx | 0
3345
3346 movl %esp, %ebx / %ebx = current stack pointer
3347
3348 lea panic_quiesce, %eax / %eax = &panic_quiesce
3349 pushl %eax / push &panic_quiesce
3350 call panic_trigger / %eax = panic_trigger()
3351 addl $4, %esp / reset stack pointer
3352
3353 vpanic_common:
3354 cmpl $0, %eax / if (%eax == 0)
3355 je 0f / goto 0f;
3356
3357 /*
3358 * If panic_trigger() was successful, we are the first to initiate a
3359 * panic: we now switch to the reserved panic_stack before continuing.
3360 */
3361 lea panic_stack, %esp / %esp = panic_stack
3362 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3363
3364 0: subl $REGSIZE, %esp / allocate struct regs
3365
3366 /*
3367 * Now that we've got everything set up, store the register values as
3368 * they were when we entered vpanic() to the designated location in
3369 * the regs structure we allocated on the stack.
3370 */
3371 #if !defined(__GNUC_AS__)
3372 movw %gs, %edx
3373 movl %edx, REGOFF_GS(%esp)
3374 movw %fs, %edx
3375 movl %edx, REGOFF_FS(%esp)
3376 movw %es, %edx
3377 movl %edx, REGOFF_ES(%esp)
3378 movw %ds, %edx
3379 movl %edx, REGOFF_DS(%esp)
3380 #else /* __GNUC_AS__ */
3381 mov %gs, %edx
3382 mov %edx, REGOFF_GS(%esp)
3383 mov %fs, %edx
3384 mov %edx, REGOFF_FS(%esp)
3385 mov %es, %edx
3386 mov %edx, REGOFF_ES(%esp)
3387 mov %ds, %edx
3388 mov %edx, REGOFF_DS(%esp)
3389 #endif /* __GNUC_AS__ */
3390 movl %edi, REGOFF_EDI(%esp)
3391 movl %esi, REGOFF_ESI(%esp)
3392 movl 16(%ebx), %ecx
3393 movl %ecx, REGOFF_EBP(%esp)
3394 movl %ebx, %ecx
3395 addl $20, %ecx
3396 movl %ecx, REGOFF_ESP(%esp)
3397 movl 8(%ebx), %ecx
3398 movl %ecx, REGOFF_EBX(%esp)
3399 movl 0(%ebx), %ecx
3400 movl %ecx, REGOFF_EDX(%esp)
3401 movl 4(%ebx), %ecx
3402 movl %ecx, REGOFF_ECX(%esp)
3403 movl 12(%ebx), %ecx
3404 movl %ecx, REGOFF_EAX(%esp)
3405 movl $0, REGOFF_TRAPNO(%esp)
3406 movl $0, REGOFF_ERR(%esp)
3407 lea vpanic, %ecx
3408 movl %ecx, REGOFF_EIP(%esp)
3409 #if !defined(__GNUC_AS__)
3410 movw %cs, %edx
3411 #else /* __GNUC_AS__ */
3412 mov %cs, %edx
3413 #endif /* __GNUC_AS__ */
3414 movl %edx, REGOFF_CS(%esp)
3415 pushfl
3416 popl %ecx
3417 #if defined(__xpv)
3418 /*
3419 * Synthesize the PS_IE bit from the event mask bit
3420 */
3421 CURTHREAD(%edx)
3422 KPREEMPT_DISABLE(%edx)
3423 EVENT_MASK_TO_IE(%edx, %ecx)
3424 CURTHREAD(%edx)
3425 KPREEMPT_ENABLE_NOKP(%edx)
3426 #endif
3427 movl %ecx, REGOFF_EFL(%esp)
3428 movl $0, REGOFF_UESP(%esp)
3429 #if !defined(__GNUC_AS__)
3430 movw %ss, %edx
3431 #else /* __GNUC_AS__ */
3432 mov %ss, %edx
3433 #endif /* __GNUC_AS__ */
3434 movl %edx, REGOFF_SS(%esp)
3435
3436 movl %esp, %ecx / %ecx = ®s
3437 pushl %eax / push on_panic_stack
3438 pushl %ecx / push ®s
3439 movl 12(%ebp), %ecx / %ecx = alist
3440 pushl %ecx / push alist
3441 movl 8(%ebp), %ecx / %ecx = format
3442 pushl %ecx / push format
3443 call panicsys / panicsys();
3444 addl $16, %esp / pop arguments
3445
3446 addl $REGSIZE, %esp
3447 popl %edx
3448 popl %ecx
3449 popl %ebx
3450 popl %eax
3451 leave
3452 ret
3453 SET_SIZE(vpanic)
3454
3455 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3456
3457 pushl %ebp / | %eip | 20
3458 movl %esp, %ebp / | %ebp | 16
3459 pushl %eax / | %eax | 12
3460 pushl %ebx / | %ebx | 8
3461 pushl %ecx / | %ecx | 4
3462 pushl %edx / | %edx | 0
3463
3464 movl %esp, %ebx / %ebx = current stack pointer
3465
3466 lea panic_quiesce, %eax / %eax = &panic_quiesce
3467 pushl %eax / push &panic_quiesce
3468 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3469 addl $4, %esp / reset stack pointer
3470 jmp vpanic_common / jump back to common code
3471
3472 SET_SIZE(dtrace_vpanic)
3473
3474 #endif /* __i386 */
3475 #endif /* __lint */
3476
3477 #if defined(__lint)
3478
3479 void
3480 hres_tick(void)
3481 {}
3482
3483 int64_t timedelta;
3484 hrtime_t hres_last_tick;
3485 volatile timestruc_t hrestime;
3486 int64_t hrestime_adj;
3487 volatile int hres_lock;
3488 hrtime_t hrtime_base;
3489
3490 #else /* __lint */
3491
3492 DGDEF3(hrestime, _MUL(2, CLONGSIZE), 8)
3493 .NWORD 0, 0
3494
3495 DGDEF3(hrestime_adj, 8, 8)
3496 .long 0, 0
3497
3498 DGDEF3(hres_last_tick, 8, 8)
3499 .long 0, 0
3500
3501 DGDEF3(timedelta, 8, 8)
3502 .long 0, 0
3503
3504 DGDEF3(hres_lock, 4, 8)
3505 .long 0
3506
3507 /*
3508 * initialized to a non zero value to make pc_gethrtime()
3509 * work correctly even before clock is initialized
3510 */
3511 DGDEF3(hrtime_base, 8, 8)
3512 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3513
3514 DGDEF3(adj_shift, 4, 4)
3515 .long ADJ_SHIFT
3516
3517 #if defined(__amd64)
3518
3519 ENTRY_NP(hres_tick)
3520 pushq %rbp
3521 movq %rsp, %rbp
3522
3523 /*
3524 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3525 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3526 * At worst, performing this now instead of under CLOCK_LOCK may
3527 * introduce some jitter in pc_gethrestime().
3528 */
3529 call *gethrtimef(%rip)
3530 movq %rax, %r8
3531
3532 leaq hres_lock(%rip), %rax
3533 movb $-1, %dl
3534 .CL1:
3535 xchgb %dl, (%rax)
3536 testb %dl, %dl
3537 jz .CL3 /* got it */
3538 .CL2:
3539 cmpb $0, (%rax) /* possible to get lock? */
3540 pause
3541 jne .CL2
3542 jmp .CL1 /* yes, try again */
3543 .CL3:
3544 /*
3545 * compute the interval since last time hres_tick was called
3546 * and adjust hrtime_base and hrestime accordingly
3547 * hrtime_base is an 8 byte value (in nsec), hrestime is
3548 * a timestruc_t (sec, nsec)
3549 */
3550 leaq hres_last_tick(%rip), %rax
3551 movq %r8, %r11
3552 subq (%rax), %r8
3553 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3554 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3555 /*
3556 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3557 */
3558 movq %r11, (%rax)
3559
3560 call __adj_hrestime
3561
3562 /*
3563 * release the hres_lock
3564 */
3565 incl hres_lock(%rip)
3566 leave
3567 ret
3568 SET_SIZE(hres_tick)
3569
3570 #elif defined(__i386)
3571
3572 ENTRY_NP(hres_tick)
3573 pushl %ebp
3574 movl %esp, %ebp
3575 pushl %esi
3576 pushl %ebx
3577
3578 /*
3579 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3580 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3581 * At worst, performing this now instead of under CLOCK_LOCK may
3582 * introduce some jitter in pc_gethrestime().
3583 */
3584 call *gethrtimef
3585 movl %eax, %ebx
3586 movl %edx, %esi
3587
3588 movl $hres_lock, %eax
3589 movl $-1, %edx
3590 .CL1:
3591 xchgb %dl, (%eax)
3592 testb %dl, %dl
3593 jz .CL3 / got it
3594 .CL2:
3595 cmpb $0, (%eax) / possible to get lock?
3596 pause
3597 jne .CL2
3598 jmp .CL1 / yes, try again
3599 .CL3:
3600 /*
3601 * compute the interval since last time hres_tick was called
3602 * and adjust hrtime_base and hrestime accordingly
3603 * hrtime_base is an 8 byte value (in nsec), hrestime is
3604 * timestruc_t (sec, nsec)
3605 */
3606
3607 lea hres_last_tick, %eax
3608
3609 movl %ebx, %edx
3610 movl %esi, %ecx
3611
3612 subl (%eax), %edx
3613 sbbl 4(%eax), %ecx
3614
3615 addl %edx, hrtime_base / add interval to hrtime_base
3616 adcl %ecx, hrtime_base+4
3617
3618 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3619
3620 /
3621 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3622 /
3623 movl %ebx, (%eax)
3624 movl %esi, 4(%eax)
3625
3626 / get hrestime at this moment. used as base for pc_gethrestime
3627 /
3628 / Apply adjustment, if any
3629 /
3630 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3631 / (max_hres_adj)
3632 /
3633 / void
3634 / adj_hrestime()
3635 / {
3636 / long long adj;
3637 /
3638 / if (hrestime_adj == 0)
3639 / adj = 0;
3640 / else if (hrestime_adj > 0) {
3641 / if (hrestime_adj < HRES_ADJ)
3642 / adj = hrestime_adj;
3643 / else
3644 / adj = HRES_ADJ;
3645 / }
3646 / else {
3647 / if (hrestime_adj < -(HRES_ADJ))
3648 / adj = -(HRES_ADJ);
3649 / else
3650 / adj = hrestime_adj;
3651 / }
3652 /
3653 / timedelta -= adj;
3654 / hrestime_adj = timedelta;
3655 / hrestime.tv_nsec += adj;
3656 /
3657 / while (hrestime.tv_nsec >= NANOSEC) {
3658 / one_sec++;
3659 / hrestime.tv_sec++;
3660 / hrestime.tv_nsec -= NANOSEC;
3661 / }
3662 / }
3663 __adj_hrestime:
3664 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3665 movl hrestime_adj+4, %edx
3666 andl %esi, %esi
3667 jne .CL4 / no
3668 andl %edx, %edx
3669 jne .CL4 / no
3670 subl %ecx, %ecx / yes, adj = 0;
3671 subl %edx, %edx
3672 jmp .CL5
3673 .CL4:
3674 subl %ecx, %ecx
3675 subl %eax, %eax
3676 subl %esi, %ecx
3677 sbbl %edx, %eax
3678 andl %eax, %eax / if (hrestime_adj > 0)
3679 jge .CL6
3680
3681 / In the following comments, HRES_ADJ is used, while in the code
3682 / max_hres_adj is used.
3683 /
3684 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3685 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3686 / on the logical equivalence of:
3687 /
3688 / !(hrestime_adj < HRES_ADJ)
3689 /
3690 / and the two step sequence:
3691 /
3692 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3693 /
3694 / which computes whether or not the least significant 32-bits
3695 / of hrestime_adj is greater than HRES_ADJ, followed by:
3696 /
3697 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3698 /
3699 / which generates a carry whenever step 1 is true or the most
3700 / significant long of the longlong hrestime_adj is non-zero.
3701
3702 movl max_hres_adj, %ecx / hrestime_adj is positive
3703 subl %esi, %ecx
3704 movl %edx, %eax
3705 adcl $-1, %eax
3706 jnc .CL7
3707 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3708 subl %edx, %edx
3709 jmp .CL5
3710
3711 / The following computation is similar to the one above.
3712 /
3713 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3714 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3715 / on the logical equivalence of:
3716 /
3717 / (hrestime_adj > -HRES_ADJ)
3718 /
3719 / and the two step sequence:
3720 /
3721 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3722 /
3723 / which means the least significant 32-bits of hrestime_adj is
3724 / greater than -HRES_ADJ, followed by:
3725 /
3726 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3727 /
3728 / which generates a carry only when step 1 is true and the most
3729 / significant long of the longlong hrestime_adj is -1.
3730
3731 .CL6: / hrestime_adj is negative
3732 movl %esi, %ecx
3733 addl max_hres_adj, %ecx
3734 movl %edx, %eax
3735 adcl $0, %eax
3736 jc .CL7
3737 xor %ecx, %ecx
3738 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3739 movl $-1, %edx
3740 jmp .CL5
3741 .CL7:
3742 movl %esi, %ecx / adj = hrestime_adj;
3743 .CL5:
3744 movl timedelta, %esi
3745 subl %ecx, %esi
3746 movl timedelta+4, %eax
3747 sbbl %edx, %eax
3748 movl %esi, timedelta
3749 movl %eax, timedelta+4 / timedelta -= adj;
3750 movl %esi, hrestime_adj
3751 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3752 addl hrestime+4, %ecx
3753
3754 movl %ecx, %eax / eax = tv_nsec
3755 1:
3756 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3757 jb .CL8 / no
3758 incl one_sec / yes, one_sec++;
3759 incl hrestime / hrestime.tv_sec++;
3760 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3761 jmp 1b / check for more seconds
3762
3763 .CL8:
3764 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3765 incl hres_lock / release the hres_lock
3766
3767 popl %ebx
3768 popl %esi
3769 leave
3770 ret
3771 SET_SIZE(hres_tick)
3772
3773 #endif /* __i386 */
3774 #endif /* __lint */
3775
3776 /*
3777 * void prefetch_smap_w(void *)
3778 *
3779 * Prefetch ahead within a linear list of smap structures.
3780 * Not implemented for ia32. Stub for compatibility.
3781 */
3782
3783 #if defined(__lint)
3784
3785 /*ARGSUSED*/
3786 void prefetch_smap_w(void *smp)
3787 {}
3788
3789 #else /* __lint */
3790
3791 ENTRY(prefetch_smap_w)
3792 rep; ret /* use 2 byte return instruction when branch target */
3793 /* AMD Software Optimization Guide - Section 6.2 */
3794 SET_SIZE(prefetch_smap_w)
3795
3796 #endif /* __lint */
3797
3798 /*
3799 * prefetch_page_r(page_t *)
3800 * issue prefetch instructions for a page_t
3801 */
3802 #if defined(__lint)
3803
3804 /*ARGSUSED*/
3805 void
3806 prefetch_page_r(void *pp)
3807 {}
3808
3809 #else /* __lint */
3810
3811 ENTRY(prefetch_page_r)
3812 rep; ret /* use 2 byte return instruction when branch target */
3813 /* AMD Software Optimization Guide - Section 6.2 */
3814 SET_SIZE(prefetch_page_r)
3815
3816 #endif /* __lint */
3817
3818 #if defined(__lint)
3819
3820 /*ARGSUSED*/
3821 int
3822 bcmp(const void *s1, const void *s2, size_t count)
3823 { return (0); }
3824
3825 #else /* __lint */
3826
3827 #if defined(__amd64)
3828
3829 ENTRY(bcmp)
3830 pushq %rbp
3831 movq %rsp, %rbp
3832 #ifdef DEBUG
3833 movq postbootkernelbase(%rip), %r11
3834 cmpq %r11, %rdi
3835 jb 0f
3836 cmpq %r11, %rsi
3837 jnb 1f
3838 0: leaq .bcmp_panic_msg(%rip), %rdi
3839 xorl %eax, %eax
3840 call panic
3841 1:
3842 #endif /* DEBUG */
3843 call memcmp
3844 testl %eax, %eax
3845 setne %dl
3846 leave
3847 movzbl %dl, %eax
3848 ret
3849 SET_SIZE(bcmp)
3850
3851 #elif defined(__i386)
3852
3853 #define ARG_S1 8
3854 #define ARG_S2 12
3855 #define ARG_LENGTH 16
3856
3857 ENTRY(bcmp)
3858 pushl %ebp
3859 movl %esp, %ebp / create new stack frame
3860 #ifdef DEBUG
3861 movl postbootkernelbase, %eax
3862 cmpl %eax, ARG_S1(%ebp)
3863 jb 0f
3864 cmpl %eax, ARG_S2(%ebp)
3865 jnb 1f
3866 0: pushl $.bcmp_panic_msg
3867 call panic
3868 1:
3869 #endif /* DEBUG */
3870
3871 pushl %edi / save register variable
3872 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3873 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3874 cmpl %eax, %ecx / if the same string
3875 je .equal / goto .equal
3876 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3877 cmpl $4, %edi / if %edi < 4
3878 jb .byte_check / goto .byte_check
3879 .align 4
3880 .word_loop:
3881 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3882 leal -4(%edi), %edi / %edi -= 4
3883 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3884 jne .word_not_equal / if not equal, goto .word_not_equal
3885 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3886 leal 4(%eax), %eax / %eax += 4 (next word)
3887 cmpl $4, %edi / if %edi >= 4
3888 jae .word_loop / goto .word_loop
3889 .byte_check:
3890 cmpl $0, %edi / if %edi == 0
3891 je .equal / goto .equal
3892 jmp .byte_loop / goto .byte_loop (checks in bytes)
3893 .word_not_equal:
3894 leal 4(%edi), %edi / %edi += 4 (post-decremented)
3895 .align 4
3896 .byte_loop:
3897 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
3898 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
3899 jne .not_equal / if not equal, goto .not_equal
3900 incl %ecx / %ecx++ (next byte)
3901 incl %eax / %eax++ (next byte)
3902 decl %edi / %edi--
3903 jnz .byte_loop / if not zero, goto .byte_loop
3904 .equal:
3905 xorl %eax, %eax / %eax = 0
3906 popl %edi / restore register variable
3907 leave / restore old stack frame
3908 ret / return (NULL)
3909 .align 4
3910 .not_equal:
3911 movl $1, %eax / return 1
3912 popl %edi / restore register variable
3913 leave / restore old stack frame
3914 ret / return (NULL)
3915 SET_SIZE(bcmp)
3916
3917 #endif /* __i386 */
3918
3919 #ifdef DEBUG
3920 .text
3921 .bcmp_panic_msg:
3922 .string "bcmp: arguments below kernelbase"
3923 #endif /* DEBUG */
3924
3925 #endif /* __lint */
3926
3927 #if defined(__lint)
3928
3929 uint_t
3930 bsrw_insn(uint16_t mask)
3931 {
3932 uint_t index = sizeof (mask) * NBBY - 1;
3933
3934 while ((mask & (1 << index)) == 0)
3935 index--;
3936 return (index);
3937 }
3938
3939 #else /* __lint */
3940
3941 #if defined(__amd64)
3942
3943 ENTRY_NP(bsrw_insn)
3944 xorl %eax, %eax
3945 bsrw %di, %ax
3946 ret
3947 SET_SIZE(bsrw_insn)
3948
3949 #elif defined(__i386)
3950
3951 ENTRY_NP(bsrw_insn)
3952 movw 4(%esp), %cx
3953 xorl %eax, %eax
3954 bsrw %cx, %ax
3955 ret
3956 SET_SIZE(bsrw_insn)
3957
3958 #endif /* __i386 */
3959 #endif /* __lint */
3960
3961 #if defined(__lint)
3962
3963 uint_t
3964 atomic_btr32(uint32_t *pending, uint_t pil)
3965 {
3966 return (*pending &= ~(1 << pil));
3967 }
3968
3969 #else /* __lint */
3970
3971 #if defined(__i386)
3972
3973 ENTRY_NP(atomic_btr32)
3974 movl 4(%esp), %ecx
3975 movl 8(%esp), %edx
3976 xorl %eax, %eax
3977 lock
3978 btrl %edx, (%ecx)
3979 setc %al
3980 ret
3981 SET_SIZE(atomic_btr32)
3982
3983 #endif /* __i386 */
3984 #endif /* __lint */
3985
3986 #if defined(__lint)
3987
3988 /*ARGSUSED*/
3989 void
3990 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
3991 uint_t arg2)
3992 {}
3993
3994 #else /* __lint */
3995
3996 #if defined(__amd64)
3997
3998 ENTRY_NP(switch_sp_and_call)
3999 pushq %rbp
4000 movq %rsp, %rbp /* set up stack frame */
4001 movq %rdi, %rsp /* switch stack pointer */
4002 movq %rdx, %rdi /* pass func arg 1 */
4003 movq %rsi, %r11 /* save function to call */
4004 movq %rcx, %rsi /* pass func arg 2 */
4005 call *%r11 /* call function */
4006 leave /* restore stack */
4007 ret
4008 SET_SIZE(switch_sp_and_call)
4009
4010 #elif defined(__i386)
4011
4012 ENTRY_NP(switch_sp_and_call)
4013 pushl %ebp
4014 mov %esp, %ebp /* set up stack frame */
4015 movl 8(%ebp), %esp /* switch stack pointer */
4016 pushl 20(%ebp) /* push func arg 2 */
4017 pushl 16(%ebp) /* push func arg 1 */
4018 call *12(%ebp) /* call function */
4019 addl $8, %esp /* pop arguments */
4020 leave /* restore stack */
4021 ret
4022 SET_SIZE(switch_sp_and_call)
4023
4024 #endif /* __i386 */
4025 #endif /* __lint */
4026
4027 #if defined(__lint)
4028
4029 void
4030 kmdb_enter(void)
4031 {}
4032
4033 #else /* __lint */
4034
4035 #if defined(__amd64)
4036
4037 ENTRY_NP(kmdb_enter)
4038 pushq %rbp
4039 movq %rsp, %rbp
4040
4041 /*
4042 * Save flags, do a 'cli' then return the saved flags
4043 */
4044 call intr_clear
4045
4046 int $T_DBGENTR
4047
4048 /*
4049 * Restore the saved flags
4050 */
4051 movq %rax, %rdi
4052 call intr_restore
4053
4054 leave
4055 ret
4056 SET_SIZE(kmdb_enter)
4057
4058 #elif defined(__i386)
4059
4060 ENTRY_NP(kmdb_enter)
4061 pushl %ebp
4062 movl %esp, %ebp
4063
4064 /*
4065 * Save flags, do a 'cli' then return the saved flags
4066 */
4067 call intr_clear
4068
4069 int $T_DBGENTR
4070
4071 /*
4072 * Restore the saved flags
4073 */
4074 pushl %eax
4075 call intr_restore
4076 addl $4, %esp
4077
4078 leave
4079 ret
4080 SET_SIZE(kmdb_enter)
4081
4082 #endif /* __i386 */
4083 #endif /* __lint */
4084
4085 #if defined(__lint)
4086
4087 void
4088 return_instr(void)
4089 {}
4090
4091 #else /* __lint */
4092
4093 ENTRY_NP(return_instr)
4094 rep; ret /* use 2 byte instruction when branch target */
4095 /* AMD Software Optimization Guide - Section 6.2 */
4096 SET_SIZE(return_instr)
4097
4098 #endif /* __lint */
4099
4100 #if defined(__lint)
4101
4102 ulong_t
4103 getflags(void)
4104 {
4105 return (0);
4106 }
4107
4108 #else /* __lint */
4109
4110 #if defined(__amd64)
4111
4112 ENTRY(getflags)
4113 pushfq
4114 popq %rax
4115 #if defined(__xpv)
4116 CURTHREAD(%rdi)
4117 KPREEMPT_DISABLE(%rdi)
4118 /*
4119 * Synthesize the PS_IE bit from the event mask bit
4120 */
4121 CURVCPU(%r11)
4122 andq $_BITNOT(PS_IE), %rax
4123 XEN_TEST_UPCALL_MASK(%r11)
4124 jnz 1f
4125 orq $PS_IE, %rax
4126 1:
4127 KPREEMPT_ENABLE_NOKP(%rdi)
4128 #endif
4129 ret
4130 SET_SIZE(getflags)
4131
4132 #elif defined(__i386)
4133
4134 ENTRY(getflags)
4135 pushfl
4136 popl %eax
4137 #if defined(__xpv)
4138 CURTHREAD(%ecx)
4139 KPREEMPT_DISABLE(%ecx)
4140 /*
4141 * Synthesize the PS_IE bit from the event mask bit
4142 */
4143 CURVCPU(%edx)
4144 andl $_BITNOT(PS_IE), %eax
4145 XEN_TEST_UPCALL_MASK(%edx)
4146 jnz 1f
4147 orl $PS_IE, %eax
4148 1:
4149 KPREEMPT_ENABLE_NOKP(%ecx)
4150 #endif
4151 ret
4152 SET_SIZE(getflags)
4153
4154 #endif /* __i386 */
4155
4156 #endif /* __lint */
4157
4158 #if defined(__lint)
4159
4160 ftrace_icookie_t
4161 ftrace_interrupt_disable(void)
4162 { return (0); }
4163
4164 #else /* __lint */
4165
4166 #if defined(__amd64)
4167
4168 ENTRY(ftrace_interrupt_disable)
4169 pushfq
4170 popq %rax
4171 CLI(%rdx)
4172 ret
4173 SET_SIZE(ftrace_interrupt_disable)
4174
4175 #elif defined(__i386)
4176
4177 ENTRY(ftrace_interrupt_disable)
4178 pushfl
4179 popl %eax
4180 CLI(%edx)
4181 ret
4182 SET_SIZE(ftrace_interrupt_disable)
4183
4184 #endif /* __i386 */
4185 #endif /* __lint */
4186
4187 #if defined(__lint)
4188
4189 /*ARGSUSED*/
4190 void
4191 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4192 {}
4193
4194 #else /* __lint */
4195
4196 #if defined(__amd64)
4197
4198 ENTRY(ftrace_interrupt_enable)
4199 pushq %rdi
4200 popfq
4201 ret
4202 SET_SIZE(ftrace_interrupt_enable)
4203
4204 #elif defined(__i386)
4205
4206 ENTRY(ftrace_interrupt_enable)
4207 movl 4(%esp), %eax
4208 pushl %eax
4209 popfl
4210 ret
4211 SET_SIZE(ftrace_interrupt_enable)
4212
4213 #endif /* __i386 */
4214 #endif /* __lint */