APUE杂记：解释器文件

November 14, 2014

一个非常常见的Python脚本如下：

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys

def main():
    """
    main method for my test script~
    """
    print sys.argv

if __name__ == '__main__':
    main()

一直以来从来没考虑过为什么在脚本的第一行要写上 #!/usr/bin/env python 这样的注释，通常的解释是这样写就知道用什么来解释这个文件了，但是也没有深究为什么。其实这是一个Unix解释器文件的写法。

对这种文件的解释是内核作为exec系统调用处理的一部分完成的。当我们执行这个脚本时，实际上发生的过程是这样的：

sh程序执行fork系统调用生成子进程
子进程执行exec系统调用，执行 /home/Documents/kongfy/args.py a b c
内核执行exec，发现该文件不是可执行格式（如ELF），作为解释器文件执行，使用第一行的解释器执行解释器，并将原参数进行位移后附加在后面（在此处内核使用pathname代替argv[0]），实际执行的程序变为 /usr/bin/env python /home/Documents/kongfy/args.py a b c

当不添加解释器注释时，错误输出如下：

kongfy@ubuntu:~/Documents$ ./args.py 
./args.py: line 1: import: command not found
./args.py: line 3: syntax error near unexpected token `('
./args.py: line 3: `def main():'

也就是说当内核找不到解释器文件的时候使用用户默认的sh执行脚本，自然会报错。

在实际解释器中执行该文件时，这一行作为注释不产生效果，也就是说，难道这就是为什么脚本语言中多采用#作为注释符的原因？

拨乱反正

一种常见的误解是#！的开头是由bash等shell程序解释的，下面从几个方面验证解释器文件确实是由内核解释的。

shell

使用bash解释带有#!的python脚本文件：

kongfy@ubuntu:~/Documents$ bash args.py 
args.py: line 4: import: command not found
args.py: line 6: syntax error near unexpected token `('
args.py: line 6: `def main():'

可见bash并不能理解解释器文件，#！在bash看来只是普通的注释而已。

strace

通过strace跟踪执行脚本文件时使用的系统调用：

kongfy@ubuntu:~/Documents$ strace ./args.py
execve("./args.py", ["./args.py", "a", "b", "c", "2"], [/* 50 vars */]) = 0
execve("/usr/lib/lightdm/lightdm/python", ["python", "./args.py", "a", "b", "c", "2"], [/* 50 vars */]) = -1 ENOENT (No such 
execve("/usr/local/sbin/python", ["python", "./args.py", "a", "b", "c", "2"], [/* 50 vars */]) = -1 ENOENT (No such file or directory)
execve("/usr/local/bin/python", ["python", "./args.py", "a", "b", "c", "2"], [/* 50 vars */]) = -1 ENOENT (No such file or directory)
execve("/usr/sbin/python", ["python", "./args.py", "a", "b", "c", "2"], [/* 50 vars */]) = -1 ENOENT (No such file or directory)
execve("/usr/bin/python", ["python", "./args.py", "a", "b", "c", "2"], [/* 50 vars */]) = 0

如果脚本文件真的是由shell解释执行的，则不应该会产生对于./args.py的execve系统调用，后面的一连串系统调用都是由/usr/bin/env生成的寻找python位置的调用。

man

执行_man 2 execve_查看execve系统调用的手册文档：

Interpreter scripts An interpreter script is a text file that has execute permission enabled and whose first line is of the form:

#! interpreter [optional-arg]

The interpreter must be a valid pathname for an executable which is not itself a script. If the filename argument of execve() specifies an interpreter script, then interpreter will be invoked with the following arguments:

interpreter [optional-arg] filename arg…

where arg… is the series of words pointed to by the argv argument of execve().

很明显，执行解释器文件是execve系统调用工作的一部分。

kernal code

实际上，对开源项目最明显最直接的验证方法必须是“read the fucking code”，只是Linux内核源码并不是那么直观，所以如果只想知道结果的话建议跳过这一小节，下面的相关内核源码贴给和我一样喜欢追根溯源的傻boy们。注：代码摘自Linux内核2.6.32.63版本的x86部分

/*
 * sys_execve() executes a new program.
 */
int sys_execve(struct pt_regs *regs)
{
	int error;
	char *filename;

	filename = getname((char __user *) regs->bx);
	error = PTR_ERR(filename);
	if (IS_ERR(filename))
		goto out;
	error = do_execve(filename,
			(char __user * __user *) regs->cx,
			(char __user * __user *) regs->dx,
			regs);
	if (error == 0) {
		/* Make sure we don't return using sysenter.. */
		set_thread_flag(TIF_IRET);
	}
	putname(filename);
out:
	return error;
}

当在用户空间执行系统调用陷入内核后，通过syscall_table找到并调用函数sys_execve，函数简单的从用户空间拷贝了filename字符串，调用do_execve函数：

/*
 * sys_execve() executes a new program.
 */
int do_execve(char * filename,
	char __user *__user *argv,
	char __user *__user *envp,
	struct pt_regs * regs)
{
	struct linux_binprm *bprm;
	struct file *file;
	struct files_struct *displaced;
	bool clear_in_exec;
	int retval;

	retval = unshare_files(&displaced);
	if (retval)
		goto out_ret;

	retval = -ENOMEM;
	bprm = kzalloc(sizeof(*bprm), GFP_KERNEL);
	if (!bprm)
		goto out_files;

	retval = prepare_bprm_creds(bprm);
	if (retval)
		goto out_free;

	retval = check_unsafe_exec(bprm);
	if (retval < 0)
		goto out_free;
	clear_in_exec = retval;
	current->in_execve = 1;

	file = open_exec(filename);
	retval = PTR_ERR(file);
	if (IS_ERR(file))
		goto out_unmark;

	sched_exec();

	bprm->file = file;
	bprm->filename = filename;
	bprm->interp = filename;

	retval = bprm_mm_init(bprm);
	if (retval)
		goto out_file;

	bprm->argc = count(argv, MAX_ARG_STRINGS);
	if ((retval = bprm->argc) < 0)
		goto out;

	bprm->envc = count(envp, MAX_ARG_STRINGS);
	if ((retval = bprm->envc) < 0)
		goto out;

	retval = prepare_binprm(bprm);
	if (retval < 0)
		goto out;

	retval = copy_strings_kernel(1, &bprm->filename, bprm);
	if (retval < 0)
		goto out;

	bprm->exec = bprm->p;
	retval = copy_strings(bprm->envc, envp, bprm);
	if (retval < 0)
		goto out;

	retval = copy_strings(bprm->argc, argv, bprm);
	if (retval < 0)
		goto out;

	current->flags &= ~PF_KTHREAD;
	retval = search_binary_handler(bprm,regs);
	if (retval < 0)
		goto out;

	/* execve succeeded */
	current->fs->in_exec = 0;
	current->in_execve = 0;
	acct_update_integrals(current);
	free_bprm(bprm);
	if (displaced)
		put_files_struct(displaced);
	return retval;

out:
	if (bprm->mm) {
		acct_arg_size(bprm, 0);
		mmput(bprm->mm);
	}

out_file:
	if (bprm->file) {
		allow_write_access(bprm->file);
		fput(bprm->file);
	}

out_unmark:
	if (clear_in_exec)
		current->fs->in_exec = 0;
	current->in_execve = 0;

out_free:
	free_bprm(bprm);

out_files:
	if (displaced)
		reset_files_struct(displaced);
out_ret:
	return retval;
}

这个函数很长，你可以慢慢品读，注意中间调用了search_binary_handler函数，该函数负责寻找实际实行该文件的方式：

/*
 * cycle the list of binary formats handler, until one recognizes the image
 */
int search_binary_handler(struct linux_binprm *bprm,struct pt_regs *regs)
{
	unsigned int depth = bprm->recursion_depth;
	int try,retval;
	struct linux_binfmt *fmt;

	/* This allows 4 levels of binfmt rewrites before failing hard. */
	if (depth > 5)
		return -ELOOP;

	retval = security_bprm_check(bprm);
	if (retval)
		return retval;
	retval = ima_bprm_check(bprm);
	if (retval)
		return retval;

	retval = audit_bprm(bprm);
	if (retval)
		return retval;

	retval = -ENOENT;
	for (try=0; try<2; try++) {
		read_lock(&binfmt_lock);
		list_for_each_entry(fmt, &formats, lh) {
			int (*fn)(struct linux_binprm *, struct pt_regs *) = fmt->load_binary;
			if (!fn)
				continue;
			if (!try_module_get(fmt->module))
				continue;
			read_unlock(&binfmt_lock);
			bprm->recursion_depth = depth + 1;
			retval = fn(bprm, regs);
			bprm->recursion_depth = depth;
			if (retval >= 0) {
				if (depth == 0)
					tracehook_report_exec(fmt, bprm, regs);
				put_binfmt(fmt);
				allow_write_access(bprm->file);
				if (bprm->file)
					fput(bprm->file);
				bprm->file = NULL;
				current->did_exec = 1;
				proc_exec_connector(current);
				return retval;
			}
			read_lock(&binfmt_lock);
			put_binfmt(fmt);
			if (retval != -ENOEXEC || bprm->mm == NULL)
				break;
			if (!bprm->file) {
				read_unlock(&binfmt_lock);
				return retval;
			}
		}
		read_unlock(&binfmt_lock);
		if (retval != -ENOEXEC || bprm->mm == NULL) {
			break;
#ifdef CONFIG_MODULES
		} else {
#define printable(c) (((c)=='\t') || ((c)=='\n') || (0x20<=(c) && (c)<=0x7e))
			if (printable(bprm->buf[0]) &&
			    printable(bprm->buf[1]) &&
			    printable(bprm->buf[2]) &&
			    printable(bprm->buf[3]))
				break; /* -ENOEXEC */
			request_module("binfmt-%04x", *(unsigned short *)(&bprm->buf[2]));
#endif
		}
	}
	return retval;
}

Linux中有多个可执行的格式，这个函数就是在这些格式中循环查找，其中一个就是script格式，对应的执行代码：

/*
 *  linux/fs/binfmt_script.c
 *
 *  Copyright (C) 1996  Martin von Löwis
 *  original #!-checking implemented by tytso.
 */

static int load_script(struct linux_binprm *bprm,struct pt_regs *regs)
{
	char *cp, *i_name, *i_arg;
	struct file *file;
	char interp[BINPRM_BUF_SIZE];
	int retval;

	if ((bprm->buf[0] != '#') || (bprm->buf[1] != '!'))
		return -ENOEXEC;
	/*
	 * This section does the #! interpretation.
	 * Sorta complicated, but hopefully it will work.  -TYT
	 */

	allow_write_access(bprm->file);
	fput(bprm->file);
	bprm->file = NULL;

	bprm->buf[BINPRM_BUF_SIZE - 1] = '\0';
	if ((cp = strchr(bprm->buf, '\n')) == NULL)
		cp = bprm->buf+BINPRM_BUF_SIZE-1;
	*cp = '\0';
	while (cp > bprm->buf) {
		cp--;
		if ((*cp == ' ') || (*cp == '\t'))
			*cp = '\0';
		else
			break;
	}
	for (cp = bprm->buf+2; (*cp == ' ') || (*cp == '\t'); cp++);
	if (*cp == '\0') 
		return -ENOEXEC; /* No interpreter name found */
	i_name = cp;
	i_arg = NULL;
	for ( ; *cp && (*cp != ' ') && (*cp != '\t'); cp++)
		/* nothing */ ;
	while ((*cp == ' ') || (*cp == '\t'))
		*cp++ = '\0';
	if (*cp)
		i_arg = cp;
	strcpy (interp, i_name);
	/*
	 * OK, we've parsed out the interpreter name and
	 * (optional) argument.
	 * Splice in (1) the interpreter's name for argv[0]
	 *           (2) (optional) argument to interpreter
	 *           (3) filename of shell script (replace argv[0])
	 *
	 * This is done in reverse order, because of how the
	 * user environment and arguments are stored.
	 */
	retval = remove_arg_zero(bprm);
	if (retval)
		return retval;
	retval = copy_strings_kernel(1, &bprm->interp, bprm);
	if (retval < 0) return retval; 
	bprm->argc++;
	if (i_arg) {
		retval = copy_strings_kernel(1, &i_arg, bprm);
		if (retval < 0) return retval; 
		bprm->argc++;
	}
	retval = copy_strings_kernel(1, &i_name, bprm);
	if (retval) return retval; 
	bprm->argc++;
	retval = bprm_change_interp(interp, bprm);
	if (retval < 0)
		return retval;

	/*
	 * OK, now restart the process with the interpreter's dentry.
	 */
	file = open_exec(interp);
	if (IS_ERR(file))
		return PTR_ERR(file);

	bprm->file = file;
	retval = prepare_binprm(bprm);
	if (retval < 0)
		return retval;
	return search_binary_handler(bprm,regs);
}

执行过程和APUE中描述的流程一致，Done。

参考资料

Unix环境高级编程
Linux man page

Fanyu Kong